531 lines
18 KiB
Python
531 lines
18 KiB
Python
# This library is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation; either
|
|
# version 2.1 of the License, or (at your option) any later version.
|
|
#
|
|
# This library is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this library; if not, write to the
|
|
# Free Software Foundation, Inc.,
|
|
# 59 Temple Place, Suite 330,
|
|
# Boston, MA 02111-1307 USA
|
|
|
|
# This file is part of urlgrabber, a high-level cross-protocol url-grabber
|
|
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
|
|
|
|
# $Id: progress.py,v 1.7 2005/08/19 21:59:07 mstenner Exp $
|
|
|
|
import sys
|
|
import time
|
|
import math
|
|
import thread
|
|
|
|
class BaseMeter:
|
|
def __init__(self):
|
|
self.update_period = 0.3 # seconds
|
|
|
|
self.filename = None
|
|
self.url = None
|
|
self.basename = None
|
|
self.text = None
|
|
self.size = None
|
|
self.start_time = None
|
|
self.last_amount_read = 0
|
|
self.last_update_time = None
|
|
self.re = RateEstimator()
|
|
|
|
def start(self, filename=None, url=None, basename=None,
|
|
size=None, now=None, text=None):
|
|
self.filename = filename
|
|
self.url = url
|
|
self.basename = basename
|
|
self.text = text
|
|
|
|
#size = None ######### TESTING
|
|
self.size = size
|
|
if not size is None: self.fsize = format_number(size) + 'B'
|
|
|
|
if now is None: now = time.time()
|
|
self.start_time = now
|
|
self.re.start(size, now)
|
|
self.last_amount_read = 0
|
|
self.last_update_time = now
|
|
self._do_start(now)
|
|
|
|
def _do_start(self, now=None):
|
|
pass
|
|
|
|
def update(self, amount_read, now=None):
|
|
# for a real gui, you probably want to override and put a call
|
|
# to your mainloop iteration function here
|
|
if now is None: now = time.time()
|
|
if (now >= self.last_update_time + self.update_period) or \
|
|
not self.last_update_time:
|
|
self.re.update(amount_read, now)
|
|
self.last_amount_read = amount_read
|
|
self.last_update_time = now
|
|
self._do_update(amount_read, now)
|
|
|
|
def _do_update(self, amount_read, now=None):
|
|
pass
|
|
|
|
def end(self, amount_read, now=None):
|
|
if now is None: now = time.time()
|
|
self.re.update(amount_read, now)
|
|
self.last_amount_read = amount_read
|
|
self.last_update_time = now
|
|
self._do_end(amount_read, now)
|
|
|
|
def _do_end(self, amount_read, now=None):
|
|
pass
|
|
|
|
class TextMeter(BaseMeter):
|
|
def __init__(self, fo=sys.stderr):
|
|
BaseMeter.__init__(self)
|
|
self.fo = fo
|
|
|
|
def _do_update(self, amount_read, now=None):
|
|
etime = self.re.elapsed_time()
|
|
fetime = format_time(etime)
|
|
fread = format_number(amount_read)
|
|
#self.size = None
|
|
if self.text is not None:
|
|
text = self.text
|
|
else:
|
|
text = self.basename
|
|
if self.size is None:
|
|
out = '\r%-60.60s %5sB %s ' % \
|
|
(text, fread, fetime)
|
|
else:
|
|
rtime = self.re.remaining_time()
|
|
frtime = format_time(rtime)
|
|
frac = self.re.fraction_read()
|
|
bar = '='*int(25 * frac)
|
|
|
|
out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \
|
|
(text, frac*100, bar, fread, frtime)
|
|
|
|
self.fo.write(out)
|
|
self.fo.flush()
|
|
|
|
def _do_end(self, amount_read, now=None):
|
|
total_time = format_time(self.re.elapsed_time())
|
|
total_size = format_number(amount_read)
|
|
if self.text is not None:
|
|
text = self.text
|
|
else:
|
|
text = self.basename
|
|
if self.size is None:
|
|
out = '\r%-60.60s %5sB %s ' % \
|
|
(text, total_size, total_time)
|
|
else:
|
|
bar = '='*25
|
|
out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ' % \
|
|
(text, 100, bar, total_size, total_time)
|
|
self.fo.write(out + '\n')
|
|
self.fo.flush()
|
|
|
|
text_progress_meter = TextMeter
|
|
|
|
class MultiFileHelper(BaseMeter):
|
|
def __init__(self, master):
|
|
BaseMeter.__init__(self)
|
|
self.master = master
|
|
|
|
def _do_start(self, now):
|
|
self.master.start_meter(self, now)
|
|
|
|
def _do_update(self, amount_read, now):
|
|
# elapsed time since last update
|
|
self.master.update_meter(self, now)
|
|
|
|
def _do_end(self, amount_read, now):
|
|
self.ftotal_time = format_time(now - self.start_time)
|
|
self.ftotal_size = format_number(self.last_amount_read)
|
|
self.master.end_meter(self, now)
|
|
|
|
def failure(self, message, now=None):
|
|
self.master.failure_meter(self, message, now)
|
|
|
|
def message(self, message):
|
|
self.master.message_meter(self, message)
|
|
|
|
class MultiFileMeter:
|
|
helperclass = MultiFileHelper
|
|
def __init__(self):
|
|
self.meters = []
|
|
self.in_progress_meters = []
|
|
self._lock = thread.allocate_lock()
|
|
self.update_period = 0.3 # seconds
|
|
|
|
self.numfiles = None
|
|
self.finished_files = 0
|
|
self.failed_files = 0
|
|
self.open_files = 0
|
|
self.total_size = None
|
|
self.failed_size = 0
|
|
self.start_time = None
|
|
self.finished_file_size = 0
|
|
self.last_update_time = None
|
|
self.re = RateEstimator()
|
|
|
|
def start(self, numfiles=None, total_size=None, now=None):
|
|
if now is None: now = time.time()
|
|
self.numfiles = numfiles
|
|
self.finished_files = 0
|
|
self.failed_files = 0
|
|
self.open_files = 0
|
|
self.total_size = total_size
|
|
self.failed_size = 0
|
|
self.start_time = now
|
|
self.finished_file_size = 0
|
|
self.last_update_time = now
|
|
self.re.start(total_size, now)
|
|
self._do_start(now)
|
|
|
|
def _do_start(self, now):
|
|
pass
|
|
|
|
def end(self, now=None):
|
|
if now is None: now = time.time()
|
|
self._do_end(now)
|
|
|
|
def _do_end(self, now):
|
|
pass
|
|
|
|
def lock(self): self._lock.acquire()
|
|
def unlock(self): self._lock.release()
|
|
|
|
###########################################################
|
|
# child meter creation and destruction
|
|
def newMeter(self):
|
|
newmeter = self.helperclass(self)
|
|
self.meters.append(newmeter)
|
|
return newmeter
|
|
|
|
def removeMeter(self, meter):
|
|
self.meters.remove(meter)
|
|
|
|
###########################################################
|
|
# child functions - these should only be called by helpers
|
|
def start_meter(self, meter, now):
|
|
if not meter in self.meters:
|
|
raise ValueError('attempt to use orphaned meter')
|
|
self._lock.acquire()
|
|
try:
|
|
if not meter in self.in_progress_meters:
|
|
self.in_progress_meters.append(meter)
|
|
self.open_files += 1
|
|
finally:
|
|
self._lock.release()
|
|
self._do_start_meter(meter, now)
|
|
|
|
def _do_start_meter(self, meter, now):
|
|
pass
|
|
|
|
def update_meter(self, meter, now):
|
|
if not meter in self.meters:
|
|
raise ValueError('attempt to use orphaned meter')
|
|
if (now >= self.last_update_time + self.update_period) or \
|
|
not self.last_update_time:
|
|
self.re.update(self._amount_read(), now)
|
|
self.last_update_time = now
|
|
self._do_update_meter(meter, now)
|
|
|
|
def _do_update_meter(self, meter, now):
|
|
pass
|
|
|
|
def end_meter(self, meter, now):
|
|
if not meter in self.meters:
|
|
raise ValueError('attempt to use orphaned meter')
|
|
self._lock.acquire()
|
|
try:
|
|
try: self.in_progress_meters.remove(meter)
|
|
except ValueError: pass
|
|
self.open_files -= 1
|
|
self.finished_files += 1
|
|
self.finished_file_size += meter.last_amount_read
|
|
finally:
|
|
self._lock.release()
|
|
self._do_end_meter(meter, now)
|
|
|
|
def _do_end_meter(self, meter, now):
|
|
pass
|
|
|
|
def failure_meter(self, meter, message, now):
|
|
if not meter in self.meters:
|
|
raise ValueError('attempt to use orphaned meter')
|
|
self._lock.acquire()
|
|
try:
|
|
try: self.in_progress_meters.remove(meter)
|
|
except ValueError: pass
|
|
self.open_files -= 1
|
|
self.failed_files += 1
|
|
if meter.size and self.failed_size is not None:
|
|
self.failed_size += meter.size
|
|
else:
|
|
self.failed_size = None
|
|
finally:
|
|
self._lock.release()
|
|
self._do_failure_meter(meter, message, now)
|
|
|
|
def _do_failure_meter(self, meter, message, now):
|
|
pass
|
|
|
|
def message_meter(self, meter, message):
|
|
pass
|
|
|
|
########################################################
|
|
# internal functions
|
|
def _amount_read(self):
|
|
tot = self.finished_file_size
|
|
for m in self.in_progress_meters:
|
|
tot += m.last_amount_read
|
|
return tot
|
|
|
|
|
|
class TextMultiFileMeter(MultiFileMeter):
|
|
def __init__(self, fo=sys.stderr):
|
|
self.fo = fo
|
|
MultiFileMeter.__init__(self)
|
|
|
|
# files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:##
|
|
def _do_update_meter(self, meter, now):
|
|
self._lock.acquire()
|
|
try:
|
|
format = "files: %3i/%-3i %3i%% data: %6.6s/%-6.6s %3i%% " \
|
|
"time: %8.8s/%8.8s"
|
|
df = self.finished_files
|
|
tf = self.numfiles or 1
|
|
pf = 100 * float(df)/tf + 0.49
|
|
dd = self.re.last_amount_read
|
|
td = self.total_size
|
|
pd = 100 * (self.re.fraction_read() or 0) + 0.49
|
|
dt = self.re.elapsed_time()
|
|
rt = self.re.remaining_time()
|
|
if rt is None: tt = None
|
|
else: tt = dt + rt
|
|
|
|
fdd = format_number(dd) + 'B'
|
|
ftd = format_number(td) + 'B'
|
|
fdt = format_time(dt, 1)
|
|
ftt = format_time(tt, 1)
|
|
|
|
out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
|
|
self.fo.write('\r' + out)
|
|
self.fo.flush()
|
|
finally:
|
|
self._lock.release()
|
|
|
|
def _do_end_meter(self, meter, now):
|
|
self._lock.acquire()
|
|
try:
|
|
format = "%-30.30s %6.6s %8.8s %9.9s"
|
|
fn = meter.basename
|
|
size = meter.last_amount_read
|
|
fsize = format_number(size) + 'B'
|
|
et = meter.re.elapsed_time()
|
|
fet = format_time(et, 1)
|
|
frate = format_number(size / et) + 'B/s'
|
|
|
|
out = '%-79.79s' % (format % (fn, fsize, fet, frate))
|
|
self.fo.write('\r' + out + '\n')
|
|
finally:
|
|
self._lock.release()
|
|
self._do_update_meter(meter, now)
|
|
|
|
def _do_failure_meter(self, meter, message, now):
|
|
self._lock.acquire()
|
|
try:
|
|
format = "%-30.30s %6.6s %s"
|
|
fn = meter.basename
|
|
if type(message) in (type(''), type(u'')):
|
|
message = message.splitlines()
|
|
if not message: message = ['']
|
|
out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
|
|
self.fo.write('\r' + out + '\n')
|
|
for m in message[1:]: self.fo.write(' ' + m + '\n')
|
|
self._lock.release()
|
|
finally:
|
|
self._do_update_meter(meter, now)
|
|
|
|
def message_meter(self, meter, message):
|
|
self._lock.acquire()
|
|
try:
|
|
pass
|
|
finally:
|
|
self._lock.release()
|
|
|
|
def _do_end(self, now):
|
|
self._do_update_meter(None, now)
|
|
self._lock.acquire()
|
|
try:
|
|
self.fo.write('\n')
|
|
self.fo.flush()
|
|
finally:
|
|
self._lock.release()
|
|
|
|
######################################################################
|
|
# support classes and functions
|
|
|
|
class RateEstimator:
|
|
def __init__(self, timescale=5.0):
|
|
self.timescale = timescale
|
|
|
|
def start(self, total=None, now=None):
|
|
if now is None: now = time.time()
|
|
self.total = total
|
|
self.start_time = now
|
|
self.last_update_time = now
|
|
self.last_amount_read = 0
|
|
self.ave_rate = None
|
|
|
|
def update(self, amount_read, now=None):
|
|
if now is None: now = time.time()
|
|
if amount_read == 0:
|
|
# if we just started this file, all bets are off
|
|
self.last_update_time = now
|
|
self.last_amount_read = 0
|
|
self.ave_rate = None
|
|
return
|
|
|
|
#print 'times', now, self.last_update_time
|
|
time_diff = now - self.last_update_time
|
|
read_diff = amount_read - self.last_amount_read
|
|
self.last_update_time = now
|
|
self.last_amount_read = amount_read
|
|
self.ave_rate = self._temporal_rolling_ave(\
|
|
time_diff, read_diff, self.ave_rate, self.timescale)
|
|
#print 'results', time_diff, read_diff, self.ave_rate
|
|
|
|
#####################################################################
|
|
# result methods
|
|
def average_rate(self):
|
|
"get the average transfer rate (in bytes/second)"
|
|
return self.ave_rate
|
|
|
|
def elapsed_time(self):
|
|
"the time between the start of the transfer and the most recent update"
|
|
return self.last_update_time - self.start_time
|
|
|
|
def remaining_time(self):
|
|
"estimated time remaining"
|
|
if not self.ave_rate or not self.total: return None
|
|
return (self.total - self.last_amount_read) / self.ave_rate
|
|
|
|
def fraction_read(self):
|
|
"""the fraction of the data that has been read
|
|
(can be None for unknown transfer size)"""
|
|
if self.total is None: return None
|
|
elif self.total == 0: return 1.0
|
|
else: return float(self.last_amount_read)/self.total
|
|
|
|
#########################################################################
|
|
# support methods
|
|
def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
|
|
"""a temporal rolling average performs smooth averaging even when
|
|
updates come at irregular intervals. This is performed by scaling
|
|
the "epsilon" according to the time since the last update.
|
|
Specifically, epsilon = time_diff / timescale
|
|
|
|
As a general rule, the average will take on a completely new value
|
|
after 'timescale' seconds."""
|
|
epsilon = time_diff / timescale
|
|
if epsilon > 1: epsilon = 1.0
|
|
return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
|
|
|
|
def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
|
|
"""perform a "rolling average" iteration
|
|
a rolling average "folds" new data into an existing average with
|
|
some weight, epsilon. epsilon must be between 0.0 and 1.0 (inclusive)
|
|
a value of 0.0 means only the old value (initial value) counts,
|
|
and a value of 1.0 means only the newest value is considered."""
|
|
|
|
try:
|
|
recent_rate = read_diff / time_diff
|
|
except ZeroDivisionError:
|
|
recent_rate = None
|
|
if last_ave is None: return recent_rate
|
|
elif recent_rate is None: return last_ave
|
|
|
|
# at this point, both last_ave and recent_rate are numbers
|
|
return epsilon * recent_rate + (1 - epsilon) * last_ave
|
|
|
|
def _round_remaining_time(self, rt, start_time=15.0):
|
|
"""round the remaining time, depending on its size
|
|
If rt is between n*start_time and (n+1)*start_time round downward
|
|
to the nearest multiple of n (for any counting number n).
|
|
If rt < start_time, round down to the nearest 1.
|
|
For example (for start_time = 15.0):
|
|
2.7 -> 2.0
|
|
25.2 -> 25.0
|
|
26.4 -> 26.0
|
|
35.3 -> 34.0
|
|
63.6 -> 60.0
|
|
"""
|
|
|
|
if rt < 0: return 0.0
|
|
shift = int(math.log(rt/start_time)/math.log(2))
|
|
rt = int(rt)
|
|
if shift <= 0: return rt
|
|
return float(int(rt) >> shift << shift)
|
|
|
|
|
|
def format_time(seconds, use_hours=0):
|
|
if seconds is None or seconds < 0:
|
|
if use_hours: return '--:--:--'
|
|
else: return '--:--'
|
|
else:
|
|
seconds = int(seconds)
|
|
minutes = seconds / 60
|
|
seconds = seconds % 60
|
|
if use_hours:
|
|
hours = minutes / 60
|
|
minutes = minutes % 60
|
|
return '%02i:%02i:%02i' % (hours, minutes, seconds)
|
|
else:
|
|
return '%02i:%02i' % (minutes, seconds)
|
|
|
|
def format_number(number, SI=0, space=' '):
|
|
"""Turn numbers into human-readable metric-like numbers"""
|
|
symbols = ['', # (none)
|
|
'k', # kilo
|
|
'M', # mega
|
|
'G', # giga
|
|
'T', # tera
|
|
'P', # peta
|
|
'E', # exa
|
|
'Z', # zetta
|
|
'Y'] # yotta
|
|
|
|
if SI: step = 1000.0
|
|
else: step = 1024.0
|
|
|
|
thresh = 999
|
|
depth = 0
|
|
max_depth = len(symbols) - 1
|
|
|
|
# we want numbers between 0 and thresh, but don't exceed the length
|
|
# of our list. In that event, the formatting will be screwed up,
|
|
# but it'll still show the right number.
|
|
while number > thresh and depth < max_depth:
|
|
depth = depth + 1
|
|
number = number / step
|
|
|
|
if type(number) == type(1) or type(number) == type(1L):
|
|
# it's an int or a long, which means it didn't get divided,
|
|
# which means it's already short enough
|
|
format = '%i%s%s'
|
|
elif number < 9.95:
|
|
# must use 9.95 for proper sizing. For example, 9.99 will be
|
|
# rounded to 10.0 with the .1f format string (which is too long)
|
|
format = '%.1f%s%s'
|
|
else:
|
|
format = '%.0f%s%s'
|
|
|
|
return(format % (float(number or 0), space, symbols[depth]))
|