From b9e581303b0d426f627e88e795f7bf20f4bdfd44 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Tue, 5 Jul 2011 19:00:53 +0200 Subject: [PATCH 001/244] [IMP] ir_cron: each job in its own thread, first stab. bzr revid: vmt@openerp.com-20110705170053-q3xgeoq21oc7dh8h --- openerp-server | 1 + openerp/addons/base/ir/ir_cron.py | 113 ++++++++++++++++++++++++------ 2 files changed, 93 insertions(+), 21 deletions(-) diff --git a/openerp-server b/openerp-server index 3586b11c7d2..347cd9f6723 100755 --- a/openerp-server +++ b/openerp-server @@ -109,6 +109,7 @@ if config['db_name']: cr.rollback() pool.get('ir.cron')._poolJobs(db.dbname) + # pool.get('ir.cron').restart(db.dbname) # jobs will start to be processed later, when start_agent below is called. cr.close() diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 054a15257ef..ba25f59525d 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -21,6 +21,8 @@ import time import logging +import threading +import psycopg2 from datetime import datetime from dateutil.relativedelta import relativedelta import netsvc @@ -74,6 +76,19 @@ class ir_cron(osv.osv, netsvc.Agent): 'doall' : lambda *a: 1 } + def f(a, b, c): + print ">>> in f" + + def expensive(a, b, c): + print ">>> in expensive" + time.sleep(80) + print ">>> out expensive" + + def expensive_2(a, b, c): + print ">>> in expensive_2" + time.sleep(80) + print ">>> out expensive_2" + def _check_args(self, cr, uid, ids, context=None): try: for this in self.browse(cr, uid, ids, context): @@ -109,45 +124,96 @@ class ir_cron(osv.osv, netsvc.Agent): except Exception, e: self._handle_callback_exception(cr, uid, model, func, args, job_id, e) - def _poolJobs(self, db_name, check=False): + def _compute_nextcall(self, job, now): + """ Compute the nextcall for a job exactly as _run_job does. """ + nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') + numbercall = job['numbercall'] + + while nextcall < now and numbercall: + if numbercall > 0: + numbercall -= 1 + if numbercall: + nextcall += _intervalTypes[job['interval_type']](job['interval_number']) + + return nextcall.strftime('%Y-%m-%d %H:%M:%S') + + def _run_job(self, cr, job, now): + """ Run a given job. """ + try: + nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') + numbercall = job['numbercall'] + + ok = False + while nextcall < now and numbercall: + if numbercall > 0: + numbercall -= 1 + if not ok or job['doall']: + self._callback(cr, job['user_id'], job['model'], job['function'], job['args'], job['id']) + if numbercall: + nextcall += _intervalTypes[job['interval_type']](job['interval_number']) + ok = True + addsql = '' + if not numbercall: + addsql = ', active=False' + cr.execute("update ir_cron set nextcall=%s, numbercall=%s"+addsql+" where id=%s", (nextcall.strftime('%Y-%m-%d %H:%M:%S'), numbercall, job['id'])) + # TODO re-schedule the master thread to nextcall if its wake-up time is later than nextcall. + # TODO NOTIFY the 'ir_cron' channel. + finally: + cr.commit() + cr.close() + + def _poolJobs(self, db_name): + # TODO remove 'check' argument from addons/base_action_rule/base_action_rule.py try: db, pool = pooler.get_db_and_pool(db_name) except: return False cr = db.cursor() try: + jobs = {} # mapping job ids to jobs for all jobs being processed. if not pool._init: now = datetime.now() cr.execute('select * from ir_cron where numbercall<>0 and active and nextcall<=now() order by priority') for job in cr.dictfetchall(): - nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') - numbercall = job['numbercall'] + task_cr = db.cursor() + task_job = None + try: + task_cr.execute('select * from ir_cron where id=%s for update nowait', (job['id'],), log_exceptions=False) + task_job = task_cr.dictfetchall()[0] + jobs[job['id']] = job + except psycopg2.OperationalError, e: + if e.pgcode == '55P03': + # Class 55: Object not in prerequisite state, 55P03: lock_not_available + continue + else: + raise + finally: + if not task_job: + task_cr.close() - ok = False - while nextcall < now and numbercall: - if numbercall > 0: - numbercall -= 1 - if not ok or job['doall']: - self._callback(cr, job['user_id'], job['model'], job['function'], job['args'], job['id']) - if numbercall: - nextcall += _intervalTypes[job['interval_type']](job['interval_number']) - ok = True - addsql = '' - if not numbercall: - addsql = ', active=False' - cr.execute("update ir_cron set nextcall=%s, numbercall=%s"+addsql+" where id=%s", (nextcall.strftime('%Y-%m-%d %H:%M:%S'), numbercall, job['id'])) - cr.commit() + task_thread = threading.Thread(target=self._run_job, name=task_job['name'], args=(task_cr, task_job, now)) + # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) + task_thread.setDaemon(False) + task_thread.start() - - cr.execute('select min(nextcall) as min_next_call from ir_cron where numbercall<>0 and active') + # Wake up time, without considering the currently processed jobs. + if jobs.keys(): + cr.execute('select min(nextcall) as min_next_call from ir_cron where numbercall<>0 and active and id not in %s', (tuple(jobs.keys()),)) + else: + cr.execute('select min(nextcall) as min_next_call from ir_cron where numbercall<>0 and active') next_call = cr.dictfetchone()['min_next_call'] if next_call: next_call = time.mktime(time.strptime(next_call, '%Y-%m-%d %H:%M:%S')) else: next_call = int(time.time()) + 3600 # if do not find active cron job from database, it will run again after 1 day - if not check: - self.setAlarm(self._poolJobs, next_call, db_name, db_name) + # Take the smallest nextcall value. + for job in jobs.values(): + nextcall = self._compute_nextcall(job, now) + if nextcall < next_call: + next_call = nextcall + + self.setAlarm(self._poolJobs, next_call, db_name, db_name) except Exception, ex: self._logger.warning('Exception in cron:', exc_info=True) @@ -156,6 +222,11 @@ class ir_cron(osv.osv, netsvc.Agent): cr.commit() cr.close() + def restart_all(self): + import openerp.models.registry + for dbname in openerp.models.registry.RegistryManager.registries: + self.restart(self, dbname) + def restart(self, dbname): self.cancel(dbname) # Reschedule cron processing job asap, but not in the current thread From 46f82438777b6a426a764c129c324289f6fd79c4 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 7 Jul 2011 15:58:43 +0200 Subject: [PATCH 002/244] [IMP] ir_cron: reschedule the main cron thread if a worker takes too long. bzr revid: vmt@openerp.com-20110707135843-z38f4r8s373ctnd2 --- openerp-server | 4 +- openerp/addons/base/ir/ir_cron.py | 75 ++++++++++++++++++++++++++----- openerp/netsvc.py | 12 +++++ 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/openerp-server b/openerp-server index 347cd9f6723..a2ec11ba996 100755 --- a/openerp-server +++ b/openerp-server @@ -108,8 +108,8 @@ if config['db_name']: openerp.tools.convert_yaml_import(cr, 'base', file(config["test_file"]), {}, 'test', True) cr.rollback() - pool.get('ir.cron')._poolJobs(db.dbname) - # pool.get('ir.cron').restart(db.dbname) # jobs will start to be processed later, when start_agent below is called. + # jobs will start to be processed later, when start_agent below is called. + pool.get('ir.cron').restart(db.dbname) cr.close() diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index ba25f59525d..9360442e922 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -76,6 +76,25 @@ class ir_cron(osv.osv, netsvc.Agent): 'doall' : lambda *a: 1 } + thread_count_lock = threading.Lock() + thread_count = 1 # maximum allowed number of thread. + + @classmethod + def get_thread_count(cls): + return cls.thread_count + + @classmethod + def inc_thread_count(cls): + cls.thread_count_lock.acquire() + cls.thread_count += 1 + cls.thread_count_lock.release() + + @classmethod + def dec_thread_count(cls): + cls.thread_count_lock.acquire() + cls.thread_count -= 1 + cls.thread_count_lock.release() + def f(a, b, c): print ">>> in f" @@ -125,7 +144,11 @@ class ir_cron(osv.osv, netsvc.Agent): self._handle_callback_exception(cr, uid, model, func, args, job_id, e) def _compute_nextcall(self, job, now): - """ Compute the nextcall for a job exactly as _run_job does. """ + """ Compute the nextcall for a job exactly as _run_job does. + + Return either the nextcall or None if it shouldn't be called. + + """ nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') numbercall = job['numbercall'] @@ -135,10 +158,12 @@ class ir_cron(osv.osv, netsvc.Agent): if numbercall: nextcall += _intervalTypes[job['interval_type']](job['interval_number']) + if not numbercall: + return None return nextcall.strftime('%Y-%m-%d %H:%M:%S') def _run_job(self, cr, job, now): - """ Run a given job. """ + """ Run a given job taking care of the repetition. """ try: nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') numbercall = job['numbercall'] @@ -156,18 +181,35 @@ class ir_cron(osv.osv, netsvc.Agent): if not numbercall: addsql = ', active=False' cr.execute("update ir_cron set nextcall=%s, numbercall=%s"+addsql+" where id=%s", (nextcall.strftime('%Y-%m-%d %H:%M:%S'), numbercall, job['id'])) - # TODO re-schedule the master thread to nextcall if its wake-up time is later than nextcall. - # TODO NOTIFY the 'ir_cron' channel. + + if numbercall: + # Reschedule our own main cron thread if necessary. + # This is really needed if this job run longer that its rescheduling period. + print ">>> advance at", nextcall + nextcall = time.mktime(time.strptime(nextcall.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) + self.reschedule_in_advance(self._poolJobs, nextcall, cr.dbname, cr.dbname) finally: cr.commit() cr.close() def _poolJobs(self, db_name): + return self._run_jobs(db_name) + + def _run_jobs(self, db_name): # TODO remove 'check' argument from addons/base_action_rule/base_action_rule.py + """ Process the cron jobs by spawning worker threads. + + This selects in database all the jobs that should be processed. It then + try to lock each of them and, if it succeeds, spawn a thread to run the + cron job (if doesn't succeed, it means another the job was already + locked to be taken care of by another thread. + + """ try: db, pool = pooler.get_db_and_pool(db_name) except: return False + print ">>> _run_jobs" cr = db.cursor() try: jobs = {} # mapping job ids to jobs for all jobs being processed. @@ -177,13 +219,16 @@ class ir_cron(osv.osv, netsvc.Agent): for job in cr.dictfetchall(): task_cr = db.cursor() task_job = None + jobs[job['id']] = job try: + # Try to lock the job... task_cr.execute('select * from ir_cron where id=%s for update nowait', (job['id'],), log_exceptions=False) task_job = task_cr.dictfetchall()[0] - jobs[job['id']] = job except psycopg2.OperationalError, e: if e.pgcode == '55P03': # Class 55: Object not in prerequisite state, 55P03: lock_not_available + # ... and fail. + print ">>>", job['name'], " is already being processed" continue else: raise @@ -191,6 +236,8 @@ class ir_cron(osv.osv, netsvc.Agent): if not task_job: task_cr.close() + # ... and succeed. + print ">>> taking care of", job['name'] task_thread = threading.Thread(target=self._run_job, name=task_job['name'], args=(task_cr, task_job, now)) # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) task_thread.setDaemon(False) @@ -202,17 +249,23 @@ class ir_cron(osv.osv, netsvc.Agent): else: cr.execute('select min(nextcall) as min_next_call from ir_cron where numbercall<>0 and active') next_call = cr.dictfetchone()['min_next_call'] + print ">>> possibility at ", next_call + + # Wake up time, taking the smallest processed job nextcall value. + for job in jobs.values(): + nextcall = self._compute_nextcall(job, now) + print ">>> or at ", nextcall + if not nextcall: + continue + if not next_call or nextcall < next_call: + next_call = nextcall + print ">>> rescheduling at", next_call + if next_call: next_call = time.mktime(time.strptime(next_call, '%Y-%m-%d %H:%M:%S')) else: next_call = int(time.time()) + 3600 # if do not find active cron job from database, it will run again after 1 day - # Take the smallest nextcall value. - for job in jobs.values(): - nextcall = self._compute_nextcall(job, now) - if nextcall < next_call: - next_call = nextcall - self.setAlarm(self._poolJobs, next_call, db_name, db_name) except Exception, ex: diff --git a/openerp/netsvc.py b/openerp/netsvc.py index 3a2add26421..26f7cbe7875 100644 --- a/openerp/netsvc.py +++ b/openerp/netsvc.py @@ -283,6 +283,18 @@ class Agent(object): for task in cls.__tasks_by_db[db_name]: task[0] = 0 + @classmethod + def reschedule_in_advance(cls, function, timestamp, db_name, *args, **kwargs): + # Cancel the previous task if any. + old_timestamp = None + if db_name in cls.__tasks_by_db: + for task in cls.__tasks_by_db[db_name]: + if task[2] == function and timestamp < task[0]: + old_timestamp = task[0] + task[0] = 0 + if not old_timestamp or timestamp < old_timestamp: + cls.setAlarm(function, timestamp, db_name, *args, **kwargs) + @classmethod def quit(cls): cls.cancel(None) From 6f5eb6b91e27f1c5aa83fd09badc3bb41fe88e3a Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Wed, 13 Jul 2011 15:49:33 +0200 Subject: [PATCH 003/244] [IMP] ir.cron: continued implementing multithreaded cron: - use a lock to protect the number of threads - the not task[0] condition in reschedule_in_advance is not really correct - but we have to remove the Agent in favor of a real cron master thread. bzr revid: vmt@openerp.com-20110713134933-gmfwddot50a3ib4k --- openerp/addons/base/ir/ir_cron.py | 61 ++++++++++++++++--------------- openerp/netsvc.py | 14 +++++-- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 9360442e922..0998b43c8b7 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -76,24 +76,18 @@ class ir_cron(osv.osv, netsvc.Agent): 'doall' : lambda *a: 1 } - thread_count_lock = threading.Lock() - thread_count = 1 # maximum allowed number of thread. + def __init__(self, pool, cr): + self.thread_count_lock = threading.Lock() + self.thread_count = 2 # maximum allowed number of thread. + super(osv.osv, self).__init__(pool, cr) - @classmethod - def get_thread_count(cls): - return cls.thread_count + def get_thread_count(self): + return self.thread_count - @classmethod - def inc_thread_count(cls): - cls.thread_count_lock.acquire() - cls.thread_count += 1 - cls.thread_count_lock.release() - - @classmethod - def dec_thread_count(cls): - cls.thread_count_lock.acquire() - cls.thread_count -= 1 - cls.thread_count_lock.release() + def dec_thread_count(self): + self.thread_count_lock.acquire() + self.thread_count -= 1 + self.thread_count_lock.release() def f(a, b, c): print ">>> in f" @@ -105,7 +99,7 @@ class ir_cron(osv.osv, netsvc.Agent): def expensive_2(a, b, c): print ">>> in expensive_2" - time.sleep(80) + time.sleep(30) print ">>> out expensive_2" def _check_args(self, cr, uid, ids, context=None): @@ -187,10 +181,16 @@ class ir_cron(osv.osv, netsvc.Agent): # This is really needed if this job run longer that its rescheduling period. print ">>> advance at", nextcall nextcall = time.mktime(time.strptime(nextcall.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) - self.reschedule_in_advance(self._poolJobs, nextcall, cr.dbname, cr.dbname) + with self.thread_count_lock: + self.reschedule_in_advance(self._poolJobs, nextcall, cr.dbname, cr.dbname) finally: cr.commit() cr.close() + with self.thread_count_lock: + self.thread_count += 1 + # reschedule the master thread in advance, using its saved next_call value. + self.reschedule_in_advance(self._poolJobs, self.next_call, cr.dbname, cr.dbname) + self.next_call = None def _poolJobs(self, db_name): return self._run_jobs(db_name) @@ -210,6 +210,7 @@ class ir_cron(osv.osv, netsvc.Agent): except: return False print ">>> _run_jobs" + self.next_call = None cr = db.cursor() try: jobs = {} # mapping job ids to jobs for all jobs being processed. @@ -217,9 +218,13 @@ class ir_cron(osv.osv, netsvc.Agent): now = datetime.now() cr.execute('select * from ir_cron where numbercall<>0 and active and nextcall<=now() order by priority') for job in cr.dictfetchall(): + print ">>>", self.get_thread_count(), "threads" + if not self.get_thread_count(): + break task_cr = db.cursor() task_job = None jobs[job['id']] = job + try: # Try to lock the job... task_cr.execute('select * from ir_cron where id=%s for update nowait', (job['id'],), log_exceptions=False) @@ -241,6 +246,7 @@ class ir_cron(osv.osv, netsvc.Agent): task_thread = threading.Thread(target=self._run_job, name=task_job['name'], args=(task_cr, task_job, now)) # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) task_thread.setDaemon(False) + self.dec_thread_count() task_thread.start() # Wake up time, without considering the currently processed jobs. @@ -251,22 +257,19 @@ class ir_cron(osv.osv, netsvc.Agent): next_call = cr.dictfetchone()['min_next_call'] print ">>> possibility at ", next_call - # Wake up time, taking the smallest processed job nextcall value. - for job in jobs.values(): - nextcall = self._compute_nextcall(job, now) - print ">>> or at ", nextcall - if not nextcall: - continue - if not next_call or nextcall < next_call: - next_call = nextcall - print ">>> rescheduling at", next_call - if next_call: next_call = time.mktime(time.strptime(next_call, '%Y-%m-%d %H:%M:%S')) else: next_call = int(time.time()) + 3600 # if do not find active cron job from database, it will run again after 1 day - self.setAlarm(self._poolJobs, next_call, db_name, db_name) + # avoid race condition: the thread rescheduled the main thread, then the main thread puts +3600. + with self.thread_count_lock: + if not self.thread_count: + print ">>> no more threads" + self.next_call = next_call + next_call = int(time.time()) + 3600 # no available thread, it will run again after 1 day + + self.reschedule_in_advance(self._poolJobs, next_call, db_name, db_name) except Exception, ex: self._logger.warning('Exception in cron:', exc_info=True) diff --git a/openerp/netsvc.py b/openerp/netsvc.py index 26f7cbe7875..5ec114f7ce1 100644 --- a/openerp/netsvc.py +++ b/openerp/netsvc.py @@ -285,14 +285,18 @@ class Agent(object): @classmethod def reschedule_in_advance(cls, function, timestamp, db_name, *args, **kwargs): + if not timestamp: + return # Cancel the previous task if any. - old_timestamp = None + old_timestamp = False if db_name in cls.__tasks_by_db: for task in cls.__tasks_by_db[db_name]: - if task[2] == function and timestamp < task[0]: - old_timestamp = task[0] + print ">>> function:", function + if task[2] == function and (not task[0] or timestamp < task[0]): + old_timestamp = True task[0] = 0 - if not old_timestamp or timestamp < old_timestamp: + if old_timestamp or db_name not in cls.__tasks_by_db or not cls.__tasks_by_db[db_name]: + print ">>> rescheduled earlier", timestamp cls.setAlarm(function, timestamp, db_name, *args, **kwargs) @classmethod @@ -306,6 +310,7 @@ class Agent(object): """ current_thread = threading.currentThread() while True: + print ">>>>> starting thread for" while cls.__tasks and cls.__tasks[0][0] < time.time(): task = heapq.heappop(cls.__tasks) timestamp, dbname, function, args, kwargs = task @@ -319,6 +324,7 @@ class Agent(object): task_thread = threading.Thread(target=function, name='netsvc.Agent.task', args=args, kwargs=kwargs) # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) task_thread.setDaemon(False) + print ">>>>> -", function.func_name task_thread.start() time.sleep(1) time.sleep(60) From b5daffc115f77d02b8e78ab6c437776f9d91e97c Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Wed, 13 Jul 2011 17:35:21 +0200 Subject: [PATCH 004/244] [IMP] registry: whene deleting a registry, also delete its cache and cron. bzr revid: vmt@openerp.com-20110713153521-isn9bllnggbxwi0z --- openerp-server | 5 +++-- openerp/modules/registry.py | 27 +++++++++++++++++++++++++-- openerp/pooler.py | 5 ----- openerp/service/web_services.py | 4 ++-- openerp/sql_db.py | 3 +-- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/openerp-server b/openerp-server index a2ec11ba996..56aa278940a 100755 --- a/openerp-server +++ b/openerp-server @@ -100,7 +100,7 @@ if not ( config["stop_after_init"] or \ if config['db_name']: for dbname in config['db_name'].split(','): - db, pool = openerp.pooler.get_db_and_pool(dbname, update_module=config['init'] or config['update'], pooljobs=False) + db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=config['init'] or config['update'], pooljobs=False) cr = db.cursor() if config["test_file"]: @@ -109,7 +109,7 @@ if config['db_name']: cr.rollback() # jobs will start to be processed later, when start_agent below is called. - pool.get('ir.cron').restart(db.dbname) + registry.start_cron_thread() cr.close() @@ -218,6 +218,7 @@ def quit(): # and would present the forced shutdown thread.join(0.05) time.sleep(0.05) + openerp.modules.registry.RegistryManager.delete_all() sys.exit(0) if config['pidfile']: diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 11f484a71a0..cb81284f6fa 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -25,6 +25,8 @@ import openerp.sql_db import openerp.osv.orm +import openerp.netsvc +import openerp.tools class Registry(object): @@ -82,6 +84,9 @@ class Registry(object): return res + def start_cron_thread(self): + self.get('ir.cron').restart(self.db.dbname) + class RegistryManager(object): """ Model registries manager. @@ -143,16 +148,34 @@ class RegistryManager(object): cr.close() if pooljobs: - registry.get('ir.cron').restart(registry.db.dbname) + registry.start_cron_thread() return registry @classmethod def delete(cls, db_name): - """ Delete the registry linked to a given database. """ + """ Delete the registry linked to a given database. + + This also cleans the associated caches. For good measure this also + cancels the associated cron job. But please note that the cron job can + be running and take some time before ending, and that you should not + remove a registry if it can still be used by some thread. So it might + be necessary to call yourself openerp.netsvc.Agent.cancel(db_name) and + and join (i.e. wait for) the thread. + + """ if db_name in cls.registries: del cls.registries[db_name] + openerp.tools.cache.clean_caches_for_db(db_name) + openerp.netsvc.Agent.cancel(db_name) + + + @classmethod + def delete_all(cls): + """ Delete all the registries. """ + for db_name in cls.registries.keys(): + cls.delete(db_name) # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/pooler.py b/openerp/pooler.py index c754385d6b3..88b60864b90 100644 --- a/openerp/pooler.py +++ b/openerp/pooler.py @@ -34,11 +34,6 @@ def get_db_and_pool(db_name, force_demo=False, status=None, update_module=False, return registry.db, registry -def delete_pool(db_name): - """Delete an existing registry.""" - RegistryManager.delete(db_name) - - def restart_pool(db_name, force_demo=False, status=None, update_module=False): """Delete an existing registry and return a database connection and a newly initialized registry.""" registry = RegistryManager.new(db_name, force_demo, status, update_module, True) diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index 40e7ccf1d69..e7757ec1e66 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -161,8 +161,8 @@ class db(netsvc.ExportService): raise Exception, e def exp_drop(self, db_name): + openerp.modules.registry.RegistryManager.delete(db_name) sql_db.close_db(db_name) - openerp.netsvc.Agent.cancel(db_name) logger = netsvc.Logger() db = sql_db.db_connect('template1') @@ -264,8 +264,8 @@ class db(netsvc.ExportService): return True def exp_rename(self, old_name, new_name): + openerp.modules.registry.RegistryManager.delete(old_name) sql_db.close_db(old_name) - openerp.netsvc.Agent.cancel(db_name) logger = netsvc.Logger() db = sql_db.db_connect('template1') diff --git a/openerp/sql_db.py b/openerp/sql_db.py index 43a508f9082..37cb8b224bb 100644 --- a/openerp/sql_db.py +++ b/openerp/sql_db.py @@ -429,9 +429,8 @@ def db_connect(db_name): return Connection(_Pool, db_name) def close_db(db_name): - """ You might want to call openerp.netsvc.Agent.cancel(db_name) along this function.""" + """ You might want to call openerp.modules.registry.RegistryManager.delete(db_name) along this function.""" _Pool.close_all(dsn(db_name)) - tools.cache.clean_caches_for_db(db_name) ct = currentThread() if hasattr(ct, 'dbname'): delattr(ct, 'dbname') From 32e830eb9963ff0fb276352ac87920ca8e2d870c Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 14 Jul 2011 13:08:09 +0200 Subject: [PATCH 005/244] [IMP] cron: removed unnecessary multi-tasks per db in Agent, some cleaning. bzr revid: vmt@openerp.com-20110714110809-sgsoev9i24589sn8 --- openerp-server | 5 +- openerp/addons/base/ir/ir_cron.py | 41 ++++----------- openerp/netsvc.py | 87 +++++++++++++------------------ 3 files changed, 48 insertions(+), 85 deletions(-) diff --git a/openerp-server b/openerp-server index 56aa278940a..d5876111c3b 100755 --- a/openerp-server +++ b/openerp-server @@ -199,7 +199,8 @@ if os.name == 'posix': signal.signal(signal.SIGQUIT, dumpstacks) def quit(): - openerp.netsvc.Agent.quit() + # stop scheduling new jobs; we will have to wait for the jobs to complete below + openerp.netsvc.Agent.cancel_all() openerp.netsvc.Server.quitAll() if config['pidfile']: os.unlink(config['pidfile']) @@ -215,7 +216,7 @@ def quit(): if thread != threading.currentThread() and not thread.isDaemon(): while thread.isAlive(): # need a busyloop here as thread.join() masks signals - # and would present the forced shutdown + # and would prevent the forced shutdown thread.join(0.05) time.sleep(0.05) openerp.modules.registry.RegistryManager.delete_all() diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 0998b43c8b7..61a8b6d1f52 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -76,10 +76,8 @@ class ir_cron(osv.osv, netsvc.Agent): 'doall' : lambda *a: 1 } - def __init__(self, pool, cr): - self.thread_count_lock = threading.Lock() - self.thread_count = 2 # maximum allowed number of thread. - super(osv.osv, self).__init__(pool, cr) + thread_count_lock = threading.Lock() + thread_count = 2 # maximum allowed number of thread. def get_thread_count(self): return self.thread_count @@ -137,25 +135,6 @@ class ir_cron(osv.osv, netsvc.Agent): except Exception, e: self._handle_callback_exception(cr, uid, model, func, args, job_id, e) - def _compute_nextcall(self, job, now): - """ Compute the nextcall for a job exactly as _run_job does. - - Return either the nextcall or None if it shouldn't be called. - - """ - nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') - numbercall = job['numbercall'] - - while nextcall < now and numbercall: - if numbercall > 0: - numbercall -= 1 - if numbercall: - nextcall += _intervalTypes[job['interval_type']](job['interval_number']) - - if not numbercall: - return None - return nextcall.strftime('%Y-%m-%d %H:%M:%S') - def _run_job(self, cr, job, now): """ Run a given job taking care of the repetition. """ try: @@ -182,20 +161,17 @@ class ir_cron(osv.osv, netsvc.Agent): print ">>> advance at", nextcall nextcall = time.mktime(time.strptime(nextcall.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) with self.thread_count_lock: - self.reschedule_in_advance(self._poolJobs, nextcall, cr.dbname, cr.dbname) + self.schedule_in_advance(nextcall, cr.dbname) finally: cr.commit() cr.close() with self.thread_count_lock: self.thread_count += 1 # reschedule the master thread in advance, using its saved next_call value. - self.reschedule_in_advance(self._poolJobs, self.next_call, cr.dbname, cr.dbname) + self.schedule_in_advance(self.next_call, cr.dbname) self.next_call = None - def _poolJobs(self, db_name): - return self._run_jobs(db_name) - - def _run_jobs(self, db_name): + def _run_jobs(self): # TODO remove 'check' argument from addons/base_action_rule/base_action_rule.py """ Process the cron jobs by spawning worker threads. @@ -205,8 +181,9 @@ class ir_cron(osv.osv, netsvc.Agent): locked to be taken care of by another thread. """ + db_name = self.pool.db.dbname try: - db, pool = pooler.get_db_and_pool(db_name) + db, pool = self.pool.db, self.pool except: return False print ">>> _run_jobs" @@ -269,7 +246,7 @@ class ir_cron(osv.osv, netsvc.Agent): self.next_call = next_call next_call = int(time.time()) + 3600 # no available thread, it will run again after 1 day - self.reschedule_in_advance(self._poolJobs, next_call, db_name, db_name) + self.schedule_in_advance(next_call, db_name) except Exception, ex: self._logger.warning('Exception in cron:', exc_info=True) @@ -286,7 +263,7 @@ class ir_cron(osv.osv, netsvc.Agent): def restart(self, dbname): self.cancel(dbname) # Reschedule cron processing job asap, but not in the current thread - self.setAlarm(self._poolJobs, time.time(), dbname, dbname) + self.schedule_in_advance(time.time(), dbname) def update_running_cron(self, cr): # Verify whether the server is already started and thus whether we need to commit diff --git a/openerp/netsvc.py b/openerp/netsvc.py index 5ec114f7ce1..7ad5778244c 100644 --- a/openerp/netsvc.py +++ b/openerp/netsvc.py @@ -37,6 +37,7 @@ from pprint import pformat # TODO modules that import netsvc only for things from loglevels must be changed to use loglevels. from loglevels import * import tools +import openerp def close_socket(sock): """ Closes a socket instance cleanly @@ -252,81 +253,65 @@ class Agent(object): * a timestamp * the database on which the task run - * the function to call - * the arguments and keyword arguments to pass to the function + * a boolean attribute specifying if the task is canceled Implementation details: - Tasks are stored as list, allowing the cancellation by setting - the timestamp to 0. + the boolean to True. - A heapq is used to store tasks, so we don't need to sort tasks ourself. """ - __tasks = [] - __tasks_by_db = {} + _wakeups = [] + _wakeup_by_db = {} _logger = logging.getLogger('netsvc.agent') - @classmethod - def setAlarm(cls, function, timestamp, db_name, *args, **kwargs): - task = [timestamp, db_name, function, args, kwargs] - heapq.heappush(cls.__tasks, task) - cls.__tasks_by_db.setdefault(db_name, []).append(task) - @classmethod def cancel(cls, db_name): - """Cancel all tasks for a given database. If None is passed, all tasks are cancelled""" - cls._logger.debug("Cancel timers for %s db", db_name or 'all') - if db_name is None: - cls.__tasks, cls.__tasks_by_db = [], {} - else: - if db_name in cls.__tasks_by_db: - for task in cls.__tasks_by_db[db_name]: - task[0] = 0 + """ Cancel next wakeup for a given database. """ + cls._logger.debug("Cancel next wake-up for database '%s'.", db_name) + if db_name in cls._wakeup_by_db: + cls._wakeup_by_db[db_name][2] = True @classmethod - def reschedule_in_advance(cls, function, timestamp, db_name, *args, **kwargs): + def cancel_all(cls): + cls._wakeups = [] + cls._wakeup_by_db = {} + + @classmethod + def schedule_in_advance(cls, timestamp, db_name): if not timestamp: return - # Cancel the previous task if any. - old_timestamp = False - if db_name in cls.__tasks_by_db: - for task in cls.__tasks_by_db[db_name]: - print ">>> function:", function - if task[2] == function and (not task[0] or timestamp < task[0]): - old_timestamp = True - task[0] = 0 - if old_timestamp or db_name not in cls.__tasks_by_db or not cls.__tasks_by_db[db_name]: + # Cancel the previous wakeup if any. + add_wakeup = False + if db_name in cls._wakeup_by_db: + task = cls._wakeup_by_db[db_name] + if task[2] or timestamp < task[0]: + add_wakeup = True + task[2] = True + else: + add_wakeup = True + if add_wakeup: print ">>> rescheduled earlier", timestamp - cls.setAlarm(function, timestamp, db_name, *args, **kwargs) - - @classmethod - def quit(cls): - cls.cancel(None) + task = [timestamp, db_name, False] + heapq.heappush(cls._wakeups, task) + cls._wakeup_by_db[db_name] = task @classmethod def runner(cls): """Neverending function (intended to be ran in a dedicated thread) that checks every 60 seconds tasks to run. TODO: make configurable """ - current_thread = threading.currentThread() while True: - print ">>>>> starting thread for" - while cls.__tasks and cls.__tasks[0][0] < time.time(): - task = heapq.heappop(cls.__tasks) - timestamp, dbname, function, args, kwargs = task - cls.__tasks_by_db[dbname].remove(task) - if not timestamp: - # null timestamp -> cancelled task + print ">>>>> cron for" + while cls._wakeups and cls._wakeups[0][0] < time.time(): + task = heapq.heappop(cls._wakeups) + timestamp, db_name, canceled = task + del cls._wakeup_by_db[db_name] + if canceled: continue - current_thread.dbname = dbname # hack hack - cls._logger.debug("Run %s.%s(*%s, **%s)", function.im_class.__name__, function.func_name, args, kwargs) - delattr(current_thread, 'dbname') - task_thread = threading.Thread(target=function, name='netsvc.Agent.task', args=args, kwargs=kwargs) - # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) - task_thread.setDaemon(False) - print ">>>>> -", function.func_name - task_thread.start() - time.sleep(1) + ir_cron = openerp.pooler.get_pool(db_name).get('ir.cron') + ir_cron._run_jobs() time.sleep(60) def start_agent(): From 2f115c21aad68d952e7a37e17fda2b69e3824ab1 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 14 Jul 2011 16:32:09 +0200 Subject: [PATCH 006/244] [IMP] cron: moved netsvc.Agent to openerp.cron. bzr revid: vmt@openerp.com-20110714143209-bebn6xg91fcrxro9 --- openerp-server | 4 +- openerp/addons/base/ir/ir_cron.py | 108 ++++++++++++------------------ openerp/modules/registry.py | 6 +- openerp/netsvc.py | 78 --------------------- 4 files changed, 46 insertions(+), 150 deletions(-) diff --git a/openerp-server b/openerp-server index d5876111c3b..f06da60236b 100755 --- a/openerp-server +++ b/openerp-server @@ -153,7 +153,7 @@ if config["translate_in"]: if config["stop_after_init"]: sys.exit(0) -openerp.netsvc.start_agent() +openerp.cron.start_master_thread() #---------------------------------------------------------- # Launch Servers @@ -200,7 +200,7 @@ if os.name == 'posix': def quit(): # stop scheduling new jobs; we will have to wait for the jobs to complete below - openerp.netsvc.Agent.cancel_all() + openerp.cron.cancel_all() openerp.netsvc.Server.quitAll() if config['pidfile']: os.unlink(config['pidfile']) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 61a8b6d1f52..f1f4547517b 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -30,6 +30,7 @@ import tools from tools.safe_eval import safe_eval as eval import pooler from osv import fields, osv +import openerp def str2tuple(s): return eval('tuple(%s)' % (s or '')) @@ -43,9 +44,8 @@ _intervalTypes = { 'minutes': lambda interval: relativedelta(minutes=interval), } -class ir_cron(osv.osv, netsvc.Agent): +class ir_cron(osv.osv): """ This is the ORM object that periodically executes actions. - Note that we use the netsvc.Agent()._logger member. """ _name = "ir.cron" _order = 'name' @@ -76,16 +76,7 @@ class ir_cron(osv.osv, netsvc.Agent): 'doall' : lambda *a: 1 } - thread_count_lock = threading.Lock() - thread_count = 2 # maximum allowed number of thread. - - def get_thread_count(self): - return self.thread_count - - def dec_thread_count(self): - self.thread_count_lock.acquire() - self.thread_count -= 1 - self.thread_count_lock.release() + _logger = logging.getLogger('cron') def f(a, b, c): print ">>> in f" @@ -160,16 +151,11 @@ class ir_cron(osv.osv, netsvc.Agent): # This is really needed if this job run longer that its rescheduling period. print ">>> advance at", nextcall nextcall = time.mktime(time.strptime(nextcall.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) - with self.thread_count_lock: - self.schedule_in_advance(nextcall, cr.dbname) + openerp.cron.schedule_in_advance(nextcall, cr.dbname) finally: cr.commit() cr.close() - with self.thread_count_lock: - self.thread_count += 1 - # reschedule the master thread in advance, using its saved next_call value. - self.schedule_in_advance(self.next_call, cr.dbname) - self.next_call = None + openerp.cron.inc_thread_count() def _run_jobs(self): # TODO remove 'check' argument from addons/base_action_rule/base_action_rule.py @@ -181,50 +167,45 @@ class ir_cron(osv.osv, netsvc.Agent): locked to be taken care of by another thread. """ - db_name = self.pool.db.dbname - try: - db, pool = self.pool.db, self.pool - except: - return False print ">>> _run_jobs" - self.next_call = None + db = self.pool.db cr = db.cursor() + db_name = db.dbname try: jobs = {} # mapping job ids to jobs for all jobs being processed. - if not pool._init: - now = datetime.now() - cr.execute('select * from ir_cron where numbercall<>0 and active and nextcall<=now() order by priority') - for job in cr.dictfetchall(): - print ">>>", self.get_thread_count(), "threads" - if not self.get_thread_count(): - break - task_cr = db.cursor() - task_job = None - jobs[job['id']] = job + now = datetime.now() + cr.execute('select * from ir_cron where numbercall<>0 and active and nextcall<=now() order by priority') + for job in cr.dictfetchall(): + print ">>>", openerp.cron.get_thread_count(), "threads" + if not openerp.cron.get_thread_count(): + break + task_cr = db.cursor() + task_job = None + jobs[job['id']] = job - try: - # Try to lock the job... - task_cr.execute('select * from ir_cron where id=%s for update nowait', (job['id'],), log_exceptions=False) - task_job = task_cr.dictfetchall()[0] - except psycopg2.OperationalError, e: - if e.pgcode == '55P03': - # Class 55: Object not in prerequisite state, 55P03: lock_not_available - # ... and fail. - print ">>>", job['name'], " is already being processed" - continue - else: - raise - finally: - if not task_job: - task_cr.close() + try: + # Try to lock the job... + task_cr.execute('select * from ir_cron where id=%s for update nowait', (job['id'],), log_exceptions=False) + task_job = task_cr.dictfetchall()[0] + except psycopg2.OperationalError, e: + if e.pgcode == '55P03': + # Class 55: Object not in prerequisite state, 55P03: lock_not_available + # ... and fail. + print ">>>", job['name'], " is already being processed" + continue + else: + raise + finally: + if not task_job: + task_cr.close() - # ... and succeed. - print ">>> taking care of", job['name'] - task_thread = threading.Thread(target=self._run_job, name=task_job['name'], args=(task_cr, task_job, now)) - # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) - task_thread.setDaemon(False) - self.dec_thread_count() - task_thread.start() + # ... and succeed. + print ">>> taking care of", job['name'] + task_thread = threading.Thread(target=self._run_job, name=task_job['name'], args=(task_cr, task_job, now)) + # force non-daemon task threads (the runner thread must be daemon, and this property is inherited by default) + task_thread.setDaemon(False) + openerp.cron.dec_thread_count() + task_thread.start() # Wake up time, without considering the currently processed jobs. if jobs.keys(): @@ -239,14 +220,7 @@ class ir_cron(osv.osv, netsvc.Agent): else: next_call = int(time.time()) + 3600 # if do not find active cron job from database, it will run again after 1 day - # avoid race condition: the thread rescheduled the main thread, then the main thread puts +3600. - with self.thread_count_lock: - if not self.thread_count: - print ">>> no more threads" - self.next_call = next_call - next_call = int(time.time()) + 3600 # no available thread, it will run again after 1 day - - self.schedule_in_advance(next_call, db_name) + openerp.cron.schedule_in_advance(next_call, db_name) except Exception, ex: self._logger.warning('Exception in cron:', exc_info=True) @@ -261,9 +235,9 @@ class ir_cron(osv.osv, netsvc.Agent): self.restart(self, dbname) def restart(self, dbname): - self.cancel(dbname) + openerp.cron.cancel(dbname) # Reschedule cron processing job asap, but not in the current thread - self.schedule_in_advance(time.time(), dbname) + openerp.cron.schedule_in_advance(time.time(), dbname) def update_running_cron(self, cr): # Verify whether the server is already started and thus whether we need to commit diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index cb81284f6fa..965ba0079b4 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -25,7 +25,7 @@ import openerp.sql_db import openerp.osv.orm -import openerp.netsvc +import openerp.cron import openerp.tools @@ -161,14 +161,14 @@ class RegistryManager(object): cancels the associated cron job. But please note that the cron job can be running and take some time before ending, and that you should not remove a registry if it can still be used by some thread. So it might - be necessary to call yourself openerp.netsvc.Agent.cancel(db_name) and + be necessary to call yourself openerp.cron.Agent.cancel(db_name) and and join (i.e. wait for) the thread. """ if db_name in cls.registries: del cls.registries[db_name] openerp.tools.cache.clean_caches_for_db(db_name) - openerp.netsvc.Agent.cancel(db_name) + openerp.cron.cancel(db_name) @classmethod diff --git a/openerp/netsvc.py b/openerp/netsvc.py index 7ad5778244c..dc5de9a4354 100644 --- a/openerp/netsvc.py +++ b/openerp/netsvc.py @@ -21,7 +21,6 @@ ############################################################################## import errno -import heapq import logging import logging.handlers import os @@ -245,83 +244,6 @@ def init_alternative_logger(): logger.addHandler(handler) logger.setLevel(logging.ERROR) -class Agent(object): - """ Singleton that keeps track of cancellable tasks to run at a given - timestamp. - - The tasks are characterised by: - - * a timestamp - * the database on which the task run - * a boolean attribute specifying if the task is canceled - - Implementation details: - - - Tasks are stored as list, allowing the cancellation by setting - the boolean to True. - - A heapq is used to store tasks, so we don't need to sort - tasks ourself. - """ - _wakeups = [] - _wakeup_by_db = {} - _logger = logging.getLogger('netsvc.agent') - - @classmethod - def cancel(cls, db_name): - """ Cancel next wakeup for a given database. """ - cls._logger.debug("Cancel next wake-up for database '%s'.", db_name) - if db_name in cls._wakeup_by_db: - cls._wakeup_by_db[db_name][2] = True - - @classmethod - def cancel_all(cls): - cls._wakeups = [] - cls._wakeup_by_db = {} - - @classmethod - def schedule_in_advance(cls, timestamp, db_name): - if not timestamp: - return - # Cancel the previous wakeup if any. - add_wakeup = False - if db_name in cls._wakeup_by_db: - task = cls._wakeup_by_db[db_name] - if task[2] or timestamp < task[0]: - add_wakeup = True - task[2] = True - else: - add_wakeup = True - if add_wakeup: - print ">>> rescheduled earlier", timestamp - task = [timestamp, db_name, False] - heapq.heappush(cls._wakeups, task) - cls._wakeup_by_db[db_name] = task - - @classmethod - def runner(cls): - """Neverending function (intended to be ran in a dedicated thread) that - checks every 60 seconds tasks to run. TODO: make configurable - """ - while True: - print ">>>>> cron for" - while cls._wakeups and cls._wakeups[0][0] < time.time(): - task = heapq.heappop(cls._wakeups) - timestamp, db_name, canceled = task - del cls._wakeup_by_db[db_name] - if canceled: - continue - ir_cron = openerp.pooler.get_pool(db_name).get('ir.cron') - ir_cron._run_jobs() - time.sleep(60) - -def start_agent(): - agent_runner = threading.Thread(target=Agent.runner, name="netsvc.Agent.runner") - # the agent runner is a typical daemon thread, that will never quit and must be - # terminated when the main process exits - with no consequence (the processing - # threads it spawns are not marked daemon) - agent_runner.setDaemon(True) - agent_runner.start() - import traceback class Server: From 4b8708fb103b5580ff3466b774412654bd7eb7a5 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 14 Jul 2011 17:11:13 +0200 Subject: [PATCH 007/244] [IMP] cron: minor cleaning. bzr revid: vmt@openerp.com-20110714151113-v07yr2rneqafbnni --- openerp-server | 4 ++-- openerp/addons/base/ir/ir_cron.py | 15 ++------------- openerp/modules/registry.py | 18 +++++++++++------- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/openerp-server b/openerp-server index f06da60236b..9e3f36cf443 100755 --- a/openerp-server +++ b/openerp-server @@ -108,8 +108,8 @@ if config['db_name']: openerp.tools.convert_yaml_import(cr, 'base', file(config["test_file"]), {}, 'test', True) cr.rollback() - # jobs will start to be processed later, when start_agent below is called. - registry.start_cron_thread() + # jobs will start to be processed later, when openerp.cron.start_master_thread below is called. + registry.schedule_cron_jobs() cr.close() diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index f1f4547517b..a49e059d526 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -105,8 +105,7 @@ class ir_cron(osv.osv): def _handle_callback_exception(self, cr, uid, model, func, args, job_id, job_exception): cr.rollback() - logger=logging.getLogger('cron') - logger.exception("Call of self.pool.get('%s').%s(cr, uid, *%r) failed in Job %s" % (model, func, args, job_id)) + self._logger.exception("Call of self.pool.get('%s').%s(cr, uid, *%r) failed in Job %s" % (model, func, args, job_id)) def _callback(self, cr, uid, model, func, args, job_id): args = str2tuple(args) @@ -229,16 +228,6 @@ class ir_cron(osv.osv): cr.commit() cr.close() - def restart_all(self): - import openerp.models.registry - for dbname in openerp.models.registry.RegistryManager.registries: - self.restart(self, dbname) - - def restart(self, dbname): - openerp.cron.cancel(dbname) - # Reschedule cron processing job asap, but not in the current thread - openerp.cron.schedule_in_advance(time.time(), dbname) - def update_running_cron(self, cr): # Verify whether the server is already started and thus whether we need to commit # immediately our changes and restart the cron agent in order to apply the change @@ -249,7 +238,7 @@ class ir_cron(osv.osv): # when the server is only starting or loading modules (hence the test on pool._init). if not self.pool._init: cr.commit() - self.restart(cr.dbname) + openerp.cron.schedule_in_advance(1, self.pool.db.dbname) def create(self, cr, uid, vals, context=None): res = super(ir_cron, self).create(cr, uid, vals, context=context) diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 965ba0079b4..9ed51044793 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -84,8 +84,16 @@ class Registry(object): return res - def start_cron_thread(self): - self.get('ir.cron').restart(self.db.dbname) + def schedule_cron_jobs(self): + """ Make the cron thread care about this registry/database jobs. + + This will initiate the cron thread to check for any pending jobs for + this registry/database as soon as possible. Then it will continously + monitors the ir.cron model for future jobs. See openerp.cron for + details. + + """ + openerp.cron.schedule_in_advance(1, self.db.dbname) class RegistryManager(object): @@ -100,7 +108,6 @@ class RegistryManager(object): # Accessed through the methods below. registries = {} - @classmethod def get(cls, db_name, force_demo=False, status=None, update_module=False, pooljobs=True): @@ -113,7 +120,6 @@ class RegistryManager(object): update_module, pooljobs) return registry - @classmethod def new(cls, db_name, force_demo=False, status=None, update_module=False, pooljobs=True): @@ -148,11 +154,10 @@ class RegistryManager(object): cr.close() if pooljobs: - registry.start_cron_thread() + registry.schedule_cron_jobs() return registry - @classmethod def delete(cls, db_name): """ Delete the registry linked to a given database. @@ -170,7 +175,6 @@ class RegistryManager(object): openerp.tools.cache.clean_caches_for_db(db_name) openerp.cron.cancel(db_name) - @classmethod def delete_all(cls): """ Delete all the registries. """ From ed910b589822c5849873f0f7c43e5cb1514561bb Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 15 Jul 2011 11:40:51 +0200 Subject: [PATCH 008/244] [IMP] cron: added some code to test the new threaded jobs, should be moved elsewhere. bzr revid: vmt@openerp.com-20110715094051-7z5v2xu91uid0jtm --- openerp/addons/base/ir/ir_cron.py | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index a49e059d526..6073489fa8a 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -44,6 +44,21 @@ _intervalTypes = { 'minutes': lambda interval: relativedelta(minutes=interval), } +JOB = { + 'function': u'f', + 'interval_type': u'minutes', + 'user_id': 1, + 'name': u'test', + 'args': False, + 'numbercall': 1, + 'nextcall': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + 'priority': 5, + 'doall': True, + 'active': True, + 'interval_number': 1, + 'model': u'ir.cron' +} + class ir_cron(osv.osv): """ This is the ORM object that periodically executes actions. """ @@ -240,6 +255,44 @@ class ir_cron(osv.osv): cr.commit() openerp.cron.schedule_in_advance(1, self.pool.db.dbname) + def _20_seconds(self, cr, uid): + print ">>> in _20_seconds" + time.sleep(20) + print ">>> out _20_seconds" + + def _80_seconds(self, cr, uid): + print ">>> in _80_seconds" + time.sleep(80) + print ">>> out _80_seconds" + + def test_0(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') + t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_0 _20_seconds A', function='_20_seconds', nextcall=t1)) + self.create(cr, uid, dict(JOB, name='test_0 _20_seconds B', function='_20_seconds', nextcall=t2)) + self.create(cr, uid, dict(JOB, name='test_0 _20_seconds C', function='_20_seconds', nextcall=t3)) + + def test_1(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_1 _20_seconds * 3', function='_20_seconds', nextcall=t1, numbercall=3)) + + def test_2(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_2 _80_seconds * 2', function='_80_seconds', nextcall=t1, numbercall=2)) + + def test_3(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') + t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_3 _80_seconds A', function='_80_seconds', nextcall=t1)) + self.create(cr, uid, dict(JOB, name='test_3 _20_seconds B', function='_20_seconds', nextcall=t2)) + self.create(cr, uid, dict(JOB, name='test_3 _20_seconds C', function='_20_seconds', nextcall=t3)) + def create(self, cr, uid, vals, context=None): res = super(ir_cron, self).create(cr, uid, vals, context=context) self.update_running_cron(cr) From f6b44ec779df0aee08837552be9d242ffa733c3c Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 15 Jul 2011 12:01:27 +0200 Subject: [PATCH 009/244] [IMP] cron: forgot to add the new openerp.cron module. bzr revid: vmt@openerp.com-20110715100127-8btlo3bluaju3em6 --- openerp/cron.py | 162 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 openerp/cron.py diff --git a/openerp/cron.py b/openerp/cron.py new file mode 100644 index 00000000000..3271e131f01 --- /dev/null +++ b/openerp/cron.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2011 OpenERP SA () +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Cron jobs scheduling + +Cron jobs are defined in the ir_cron table/model. This module deals with all +cron jobs, for all databases of a single OpenERP server instance. + +It defines a single master thread that will spawn (a bounded number of) +threads to process individual cron jobs. + +""" + +import heapq +import logging +import threading +import time + +import openerp + +""" Singleton that keeps track of cancellable tasks to run at a given + timestamp. + + The tasks are characterised by: + + * a timestamp + * the database on which the task run + * a boolean attribute specifying if the task is canceled + + Implementation details: + + - Tasks are stored as list, allowing the cancellation by setting + the boolean to True. + - A heapq is used to store tasks, so we don't need to sort + tasks ourself. +""" + +# Heapq of database wake-ups. Note that 'database wake-up' meaning is in +# the context of the cron management. This is not about loading a database +# or otherwise making anything about it. +_wakeups = [] # TODO protect this variable with a lock? + +# Mapping of database names to the wake-up defined in the heapq, +# so that we can cancel the wake-up without messing with the heapq +# internal structure. +_wakeup_by_db = {} + +_logger = logging.getLogger('cron') + +_thread_count_lock = threading.Lock() + +# Maximum number of threads allowed to process cron jobs concurrently. +_thread_count = 2 + + +def get_thread_count(): + return _thread_count + + +def inc_thread_count(): + global _thread_count + with _thread_count_lock: + _thread_count += 1 + + +def dec_thread_count(): + global _thread_count + with _thread_count_lock: + _thread_count -= 1 + + +def cancel(db_name): + """ Cancel the next wake-up of a given database, if any. """ + _logger.debug("Cancel next wake-up for database '%s'.", db_name) + if db_name in _wakeup_by_db: + _wakeup_by_db[db_name][2] = True + + +def cancel_all(): + """ Cancel all database wake-ups. """ + global _wakeups + global _wakeup_by_db + _wakeups = [] + _wakeup_by_db = {} + + +def schedule_in_advance(timestamp, db_name): + """ Schedule a wake-up for a new database. + + If an earlier wake-up is already defined, the new wake-up is discarded. + If another wake-up is defined, it is discarded. + + """ + if not timestamp: + return + # Cancel the previous wakeup if any. + add_wakeup = False + if db_name in _wakeup_by_db: + task = _wakeup_by_db[db_name] + if task[2] or timestamp < task[0]: + add_wakeup = True + task[2] = True + else: + add_wakeup = True + if add_wakeup: + task = [timestamp, db_name, False] + heapq.heappush(_wakeups, task) + _wakeup_by_db[db_name] = task + + +def runner(): + """Neverending function (intended to be ran in a dedicated thread) that + checks every 60 seconds the next database wake-up. TODO: make configurable + """ + while True: + while _wakeups and _wakeups[0][0] < time.time() and get_thread_count(): + task = heapq.heappop(_wakeups) + timestamp, db_name, canceled = task + if canceled: + continue + task[2] = True + registry = openerp.pooler.get_pool(db_name) + if not registry._init: + registry['ir.cron']._run_jobs() + if _wakeups and get_thread_count(): + time.sleep(min(60, _wakeups[0][0] - time.time())) + else: + time.sleep(60) + + +def start_master_thread(): + """ Start the above runner function in a daemon thread. + + The thread is a typical daemon thread: it will never quit and must be + terminated when the main process exits - with no consequence (the processing + threads it spawns are not marked daemon). + + """ + t = threading.Thread(target=runner, name="openerp.cron.master_thread") + t.setDaemon(True) + t.start() + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: From faf2863a3595c9a655a5e7152f2a351e3965e479 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 15 Jul 2011 13:38:45 +0200 Subject: [PATCH 010/244] [IMP] cron: bracketed the jobs heap/dict with a lock. bzr revid: vmt@openerp.com-20110715113845-zokj6cf6z0adj6h4 --- openerp/addons/base/ir/ir_cron.py | 2 +- openerp/cron.py | 68 ++++++++++++++++++------------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 6073489fa8a..2cdb15b4990 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -164,7 +164,7 @@ class ir_cron(osv.osv): # Reschedule our own main cron thread if necessary. # This is really needed if this job run longer that its rescheduling period. print ">>> advance at", nextcall - nextcall = time.mktime(time.strptime(nextcall.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')) + nextcall = time.mktime(nextcall.timetuple()) openerp.cron.schedule_in_advance(nextcall, cr.dbname) finally: cr.commit() diff --git a/openerp/cron.py b/openerp/cron.py index 3271e131f01..79347fbe178 100644 --- a/openerp/cron.py +++ b/openerp/cron.py @@ -66,6 +66,11 @@ _wakeup_by_db = {} _logger = logging.getLogger('cron') +# We could use a simple (non-reentrant) lock if the runner function below +# was more fine-grained, but we are fine with the loop owning the lock +# while spawning a few threads. +_wakeups_lock = threading.RLock() + _thread_count_lock = threading.Lock() # Maximum number of threads allowed to process cron jobs concurrently. @@ -91,16 +96,18 @@ def dec_thread_count(): def cancel(db_name): """ Cancel the next wake-up of a given database, if any. """ _logger.debug("Cancel next wake-up for database '%s'.", db_name) - if db_name in _wakeup_by_db: - _wakeup_by_db[db_name][2] = True + with _wakeups_lock: + if db_name in _wakeup_by_db: + _wakeup_by_db[db_name][2] = True def cancel_all(): """ Cancel all database wake-ups. """ global _wakeups global _wakeup_by_db - _wakeups = [] - _wakeup_by_db = {} + with _wakeups_lock: + _wakeups = [] + _wakeup_by_db = {} def schedule_in_advance(timestamp, db_name): @@ -112,19 +119,20 @@ def schedule_in_advance(timestamp, db_name): """ if not timestamp: return - # Cancel the previous wakeup if any. - add_wakeup = False - if db_name in _wakeup_by_db: - task = _wakeup_by_db[db_name] - if task[2] or timestamp < task[0]: + with _wakeups_lock: + # Cancel the previous wakeup if any. + add_wakeup = False + if db_name in _wakeup_by_db: + task = _wakeup_by_db[db_name] + if task[2] or timestamp < task[0]: + add_wakeup = True + task[2] = True + else: add_wakeup = True - task[2] = True - else: - add_wakeup = True - if add_wakeup: - task = [timestamp, db_name, False] - heapq.heappush(_wakeups, task) - _wakeup_by_db[db_name] = task + if add_wakeup: + task = [timestamp, db_name, False] + heapq.heappush(_wakeups, task) + _wakeup_by_db[db_name] = task def runner(): @@ -132,19 +140,21 @@ def runner(): checks every 60 seconds the next database wake-up. TODO: make configurable """ while True: - while _wakeups and _wakeups[0][0] < time.time() and get_thread_count(): - task = heapq.heappop(_wakeups) - timestamp, db_name, canceled = task - if canceled: - continue - task[2] = True - registry = openerp.pooler.get_pool(db_name) - if not registry._init: - registry['ir.cron']._run_jobs() - if _wakeups and get_thread_count(): - time.sleep(min(60, _wakeups[0][0] - time.time())) - else: - time.sleep(60) + with _wakeups_lock: + while _wakeups and _wakeups[0][0] < time.time() and get_thread_count(): + task = heapq.heappop(_wakeups) + timestamp, db_name, canceled = task + if canceled: + continue + task[2] = True + registry = openerp.pooler.get_pool(db_name) + if not registry._init: + registry['ir.cron']._run_jobs() + amount = 60 + with _wakeups_lock: + if _wakeups and get_thread_count(): + amount = min(60, _wakeups[0][0] - time.time()) + time.sleep(amount) def start_master_thread(): From ed1b2a92cad8058a30fd8ba65ef9137b9cd23554 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Mon, 8 Aug 2011 15:05:02 +0200 Subject: [PATCH 011/244] [IMP] cron: comments and docstrings. bzr revid: vmt@openerp.com-20110808130502-htps768jts63m9sx --- openerp/addons/base/ir/ir_cron.py | 2 +- openerp/cron.py | 62 +++++++++++++++++-------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 2cdb15b4990..04486cafdd0 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -162,7 +162,7 @@ class ir_cron(osv.osv): if numbercall: # Reschedule our own main cron thread if necessary. - # This is really needed if this job run longer that its rescheduling period. + # This is really needed if this job runs longer than its rescheduling period. print ">>> advance at", nextcall nextcall = time.mktime(nextcall.timetuple()) openerp.cron.schedule_in_advance(nextcall, cr.dbname) diff --git a/openerp/cron.py b/openerp/cron.py index 79347fbe178..f6e8a5d49a2 100644 --- a/openerp/cron.py +++ b/openerp/cron.py @@ -28,6 +28,12 @@ cron jobs, for all databases of a single OpenERP server instance. It defines a single master thread that will spawn (a bounded number of) threads to process individual cron jobs. +The thread runs forever, checking every 60 seconds for new +'database wake-ups'. It maintains a heapq of database wake-ups. At each +wake-up, it will call ir_cron._run_jobs() for the given database. _run_jobs +will check the jobs defined in the ir_cron table and spawn accordingly threads +to process them. + """ import heapq @@ -37,64 +43,61 @@ import time import openerp -""" Singleton that keeps track of cancellable tasks to run at a given - timestamp. - - The tasks are characterised by: - - * a timestamp - * the database on which the task run - * a boolean attribute specifying if the task is canceled - - Implementation details: - - - Tasks are stored as list, allowing the cancellation by setting - the boolean to True. - - A heapq is used to store tasks, so we don't need to sort - tasks ourself. -""" - # Heapq of database wake-ups. Note that 'database wake-up' meaning is in # the context of the cron management. This is not about loading a database # or otherwise making anything about it. -_wakeups = [] # TODO protect this variable with a lock? +# Each element is a triple (timestamp, database-name, boolean). The boolean +# specifies if the wake-up is canceled (so a wake-up can be canceled without +# relying on the heapq implementation detail; no need to remove the job from +# the heapq). +_wakeups = [] # Mapping of database names to the wake-up defined in the heapq, # so that we can cancel the wake-up without messing with the heapq -# internal structure. +# internal structure: lookup the wake-up by database-name, then set +# its third element to True. _wakeup_by_db = {} -_logger = logging.getLogger('cron') - +# Re-entrant lock to protect the above _wakeups and _wakeup_by_db variables. # We could use a simple (non-reentrant) lock if the runner function below # was more fine-grained, but we are fine with the loop owning the lock # while spawning a few threads. _wakeups_lock = threading.RLock() +# Maximum number of threads allowed to process cron jobs concurrently. +_thread_count = 2 # TODO make it configurable + +# A (non re-entrant) lock to protect the above _thread_count variable. _thread_count_lock = threading.Lock() -# Maximum number of threads allowed to process cron jobs concurrently. -_thread_count = 2 +_logger = logging.getLogger('cron') def get_thread_count(): + """ Return the number of available threads. """ return _thread_count def inc_thread_count(): + """ Increment by the number of available threads. """ global _thread_count with _thread_count_lock: _thread_count += 1 def dec_thread_count(): + """ Decrement by the number of available threads. """ global _thread_count with _thread_count_lock: _thread_count -= 1 def cancel(db_name): - """ Cancel the next wake-up of a given database, if any. """ + """ Cancel the next wake-up of a given database, if any. + + :param db_name: database name for which the wake-up is canceled. + + """ _logger.debug("Cancel next wake-up for database '%s'.", db_name) with _wakeups_lock: if db_name in _wakeup_by_db: @@ -111,16 +114,20 @@ def cancel_all(): def schedule_in_advance(timestamp, db_name): - """ Schedule a wake-up for a new database. + """ Schedule a new wake-up for a database. If an earlier wake-up is already defined, the new wake-up is discarded. - If another wake-up is defined, it is discarded. + If another wake-up is defined, that wake-up is discarded and the new one + is scheduled. + + :param db_name: database name for which a new wake-up is scheduled. + :param timestamp: when the wake-up is scheduled. """ if not timestamp: return with _wakeups_lock: - # Cancel the previous wakeup if any. + # Cancel the previous wake-up if any. add_wakeup = False if db_name in _wakeup_by_db: task = _wakeup_by_db[db_name] @@ -152,6 +159,7 @@ def runner(): registry['ir.cron']._run_jobs() amount = 60 with _wakeups_lock: + # Sleep less than 60s if the next known wake-up will happen before. if _wakeups and get_thread_count(): amount = min(60, _wakeups[0][0] - time.time()) time.sleep(amount) From d803d9192be06bc10a86ceb6054ab6ba2ce1dd4f Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Mon, 8 Aug 2011 15:54:53 +0200 Subject: [PATCH 012/244] [IMP] cron: the maximum number of cron threads is configurable. bzr revid: vmt@openerp.com-20110808135453-qdlhkyupb6803jln --- openerp/conf/__init__.py | 6 ++++++ openerp/cron.py | 14 +++++--------- openerp/tools/config.py | 8 +++++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/openerp/conf/__init__.py b/openerp/conf/__init__.py index 7de8070f9e2..bc8ee98ad43 100644 --- a/openerp/conf/__init__.py +++ b/openerp/conf/__init__.py @@ -32,4 +32,10 @@ of paths. import deprecation +# Maximum number of threads processing concurrently cron jobs. +# Access to this variable must be thread-safe; they have to be done +# through the functions in openerp.cron. +max_cron_threads = 4 # Actually the default value here is meaningless, + # look at tools.config for the default value. + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/cron.py b/openerp/cron.py index f6e8a5d49a2..8879c73ce62 100644 --- a/openerp/cron.py +++ b/openerp/cron.py @@ -64,10 +64,8 @@ _wakeup_by_db = {} # while spawning a few threads. _wakeups_lock = threading.RLock() -# Maximum number of threads allowed to process cron jobs concurrently. -_thread_count = 2 # TODO make it configurable - -# A (non re-entrant) lock to protect the above _thread_count variable. +# A (non re-entrant) lock to protect the openerp.conf.max_cron_threads +# variable. _thread_count_lock = threading.Lock() _logger = logging.getLogger('cron') @@ -75,21 +73,19 @@ _logger = logging.getLogger('cron') def get_thread_count(): """ Return the number of available threads. """ - return _thread_count + return openerp.conf.max_cron_threads def inc_thread_count(): """ Increment by the number of available threads. """ - global _thread_count with _thread_count_lock: - _thread_count += 1 + openerp.conf.max_cron_threads += 1 def dec_thread_count(): """ Decrement by the number of available threads. """ - global _thread_count with _thread_count_lock: - _thread_count -= 1 + openerp.conf.max_cron_threads -= 1 def cancel(db_name): diff --git a/openerp/tools/config.py b/openerp/tools/config.py index 18f40e0011f..af8ad8cdbc7 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -24,6 +24,7 @@ import optparse import os import sys import openerp +import openerp.conf import openerp.loglevels as loglevels import logging import openerp.release as release @@ -250,6 +251,9 @@ class configmanager(object): "osv_memory tables. This is a decimal value expressed in hours, " "and the default is 1 hour.", type="float") + group.add_option("--max-cron-threads", dest="max_cron_threads", my_default=4, + help="Maximum number of threads processing concurrently cron jobs.", + type="int") parser.add_option_group(group) # Copy all optparse options (i.e. MyOption) into self.options. @@ -335,7 +339,7 @@ class configmanager(object): 'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog', 'list_db', 'xmlrpcs', 'test_file', 'test_disable', 'test_commit', 'test_report_directory', - 'osv_memory_count_limit', 'osv_memory_age_limit', + 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', ] for arg in keys: @@ -417,6 +421,8 @@ class configmanager(object): if opt.save: self.save() + openerp.conf.max_cron_threads = self.options['max_cron_threads'] + def _generate_pgpassfile(self): """ Generate the pgpass file with the parameters from the command line (db_host, db_user, From f23ec137ca93456a2975582a30ccba792cebbcf5 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Mon, 8 Aug 2011 16:05:24 +0200 Subject: [PATCH 013/244] [IMP] cron: the thread-safe variable is located inside openerp.cron; the configuration variable is just read once. bzr revid: vmt@openerp.com-20110808140524-xj8sdm43upp4jr64 --- openerp/conf/__init__.py | 2 -- openerp/cron.py | 20 +++++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/openerp/conf/__init__.py b/openerp/conf/__init__.py index bc8ee98ad43..6ff832b6551 100644 --- a/openerp/conf/__init__.py +++ b/openerp/conf/__init__.py @@ -33,8 +33,6 @@ of paths. import deprecation # Maximum number of threads processing concurrently cron jobs. -# Access to this variable must be thread-safe; they have to be done -# through the functions in openerp.cron. max_cron_threads = 4 # Actually the default value here is meaningless, # look at tools.config for the default value. diff --git a/openerp/cron.py b/openerp/cron.py index 8879c73ce62..7f96a7d499d 100644 --- a/openerp/cron.py +++ b/openerp/cron.py @@ -34,6 +34,9 @@ wake-up, it will call ir_cron._run_jobs() for the given database. _run_jobs will check the jobs defined in the ir_cron table and spawn accordingly threads to process them. +This module behavior depends on the following configuration variable: +openerp.conf.max_cron_threads. + """ import heapq @@ -64,8 +67,11 @@ _wakeup_by_db = {} # while spawning a few threads. _wakeups_lock = threading.RLock() -# A (non re-entrant) lock to protect the openerp.conf.max_cron_threads -# variable. +# Maximum number of threads allowed to process cron jobs concurrently. This +# variable is set by start_master_thread using openerp.conf.max_cron_threads. +_thread_count = None + +# A (non re-entrant) lock to protect the above _thread_count variable. _thread_count_lock = threading.Lock() _logger = logging.getLogger('cron') @@ -73,19 +79,21 @@ _logger = logging.getLogger('cron') def get_thread_count(): """ Return the number of available threads. """ - return openerp.conf.max_cron_threads + return _thread_count def inc_thread_count(): """ Increment by the number of available threads. """ + global _thread_count with _thread_count_lock: - openerp.conf.max_cron_threads += 1 + _thread_count += 1 def dec_thread_count(): """ Decrement by the number of available threads. """ + global _thread_count with _thread_count_lock: - openerp.conf.max_cron_threads -= 1 + _thread_count -= 1 def cancel(db_name): @@ -169,6 +177,8 @@ def start_master_thread(): threads it spawns are not marked daemon). """ + global _thread_count + _thread_count = openerp.conf.max_cron_threads t = threading.Thread(target=runner, name="openerp.cron.master_thread") t.setDaemon(True) t.start() From 711554a4db33ee726ca820b5b7d7917d44cc3711 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Mon, 8 Aug 2011 16:59:31 +0200 Subject: [PATCH 014/244] [IMP] cron: comments, docstrings, help messages. bzr revid: vmt@openerp.com-20110808145931-ms0vpvg8uj72kyd0 --- openerp/addons/base/ir/ir_cron.py | 76 ++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 04486cafdd0..84c9c5e837c 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -60,7 +60,7 @@ JOB = { } class ir_cron(osv.osv): - """ This is the ORM object that periodically executes actions. + """ Model describing cron jobs (also called actions or tasks). """ _name = "ir.cron" _order = 'name' @@ -71,13 +71,13 @@ class ir_cron(osv.osv): 'interval_number': fields.integer('Interval Number',help="Repeat every x."), 'interval_type': fields.selection( [('minutes', 'Minutes'), ('hours', 'Hours'), ('work_days','Work Days'), ('days', 'Days'),('weeks', 'Weeks'), ('months', 'Months')], 'Interval Unit'), - 'numbercall': fields.integer('Number of Calls', help='Number of time the function is called,\na negative number indicates no limit'), - 'doall' : fields.boolean('Repeat Missed', help="Enable this if you want to execute missed occurences as soon as the server restarts."), - 'nextcall' : fields.datetime('Next Execution Date', required=True, help="Next planned execution date for this scheduler"), - 'model': fields.char('Object', size=64, help="Name of object whose function will be called when this scheduler will run. e.g. 'res.partener'"), - 'function': fields.char('Function', size=64, help="Name of the method to be called on the object when this scheduler is executed."), - 'args': fields.text('Arguments', help="Arguments to be passed to the method. e.g. (uid,)"), - 'priority': fields.integer('Priority', help='0=Very Urgent\n10=Not urgent') + 'numbercall': fields.integer('Number of Calls', help='How many times the method is called,\na negative number indicates no limit.'), + 'doall' : fields.boolean('Repeat Missed', help="Specify if missed occurrences should be executed when the server restarts."), + 'nextcall' : fields.datetime('Next Execution Date', required=True, help="Next planned execution date for this job."), + 'model': fields.char('Object', size=64, help="Model name on which the method to be called is located, e.g. 'res.partner'."), + 'function': fields.char('Method', size=64, help="Name of the method to be called when this job is processed."), + 'args': fields.text('Arguments', help="Arguments to be passed to the method, e.g. (uid,)."), + 'priority': fields.integer('Priority', help='The priority of the job, as an integer: 0 means higher priority, 10 means lower priority.') } _defaults = { @@ -118,30 +118,58 @@ class ir_cron(osv.osv): (_check_args, 'Invalid arguments', ['args']), ] - def _handle_callback_exception(self, cr, uid, model, func, args, job_id, job_exception): - cr.rollback() - self._logger.exception("Call of self.pool.get('%s').%s(cr, uid, *%r) failed in Job %s" % (model, func, args, job_id)) + def _handle_callback_exception(self, cr, uid, model_name, method_name, args, job_id, job_exception): + """ Method called when an exception is raised by a job. - def _callback(self, cr, uid, model, func, args, job_id): + Simply logs the exception and rollback the transaction. + + :param model_name: model name on which the job method is located. + :param method_name: name of the method to call when this job is processed. + :param args: arguments of the method (without the usual self, cr, uid). + :param job_id: job id. + :param job_exception: exception raised by the job. + + """ + cr.rollback() + self._logger.exception("Call of self.pool.get('%s').%s(cr, uid, *%r) failed in Job %s" % (model_name, method_name, args, job_id)) + + def _callback(self, cr, uid, model_name, method_name, args, job_id): + """ Run the method associated to a given job + + It takes care of logging and exception handling. + + :param model_name: model name on which the job method is located. + :param method_name: name of the method to call when this job is processed. + :param args: arguments of the method (without the usual self, cr, uid). + :param job_id: job id. + """ args = str2tuple(args) - m = self.pool.get(model) - if m and hasattr(m, func): - f = getattr(m, func) + model = self.pool.get(model_name) + if model and hasattr(model, method_name): + method = getattr(model, method_name) try: - netsvc.log('cron', (cr.dbname,uid,'*',model,func)+tuple(args), channel=logging.DEBUG, + netsvc.log('cron', (cr.dbname,uid,'*',model_name,method_name)+tuple(args), channel=logging.DEBUG, depth=(None if self._logger.isEnabledFor(logging.DEBUG_RPC_ANSWER) else 1), fn='object.execute') logger = logging.getLogger('execution time') if logger.isEnabledFor(logging.DEBUG): start_time = time.time() - f(cr, uid, *args) + method(cr, uid, *args) if logger.isEnabledFor(logging.DEBUG): end_time = time.time() - logger.log(logging.DEBUG, '%.3fs (%s, %s)' % (end_time - start_time, model, func)) + logger.log(logging.DEBUG, '%.3fs (%s, %s)' % (end_time - start_time, model_name, method_name)) except Exception, e: - self._handle_callback_exception(cr, uid, model, func, args, job_id, e) + self._handle_callback_exception(cr, uid, model_name, method_name, args, job_id, e) def _run_job(self, cr, job, now): - """ Run a given job taking care of the repetition. """ + """ Run a given job taking care of the repetition. + + The cursor has a lock on the job (aquired by _run_jobs()) and this + method is run in a worker thread (spawned by _run_jobs())). + + :param job: job to be run (as a dictionary). + :param now: timestamp (result of datetime.now(), no need to call it multiple time). + + """ try: nextcall = datetime.strptime(job['nextcall'], '%Y-%m-%d %H:%M:%S') numbercall = job['numbercall'] @@ -176,10 +204,13 @@ class ir_cron(osv.osv): """ Process the cron jobs by spawning worker threads. This selects in database all the jobs that should be processed. It then - try to lock each of them and, if it succeeds, spawn a thread to run the - cron job (if doesn't succeed, it means another the job was already + tries to lock each of them and, if it succeeds, spawns a thread to run + the cron job (if it doesn't succeed, it means the job was already locked to be taken care of by another thread. + The cursor used to lock the job in database is given to the worker + thread (which has to close it itself). + """ print ">>> _run_jobs" db = self.pool.db @@ -244,6 +275,7 @@ class ir_cron(osv.osv): cr.close() def update_running_cron(self, cr): + """ Schedule as soon as possible a wake-up for this database. """ # Verify whether the server is already started and thus whether we need to commit # immediately our changes and restart the cron agent in order to apply the change # immediately. The commit() is needed because as soon as the cron is (re)started it From a7e6a5bd4885366b89a42384e87746568d38a464 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Mon, 8 Aug 2011 17:20:24 +0200 Subject: [PATCH 015/244] [IMP] cron: moved testing code to some other file. That code is useful for manual testing but is not called by the normal testing framework. Its import is commented out. bzr revid: vmt@openerp.com-20110808152024-23plfx82d35ok8ua --- openerp/addons/base/__init__.py | 1 + openerp/addons/base/ir/ir_cron.py | 67 ------------------ openerp/addons/base/test/__init__.py | 25 +++++++ openerp/addons/base/test/test_ir_cron.py | 87 ++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 67 deletions(-) create mode 100644 openerp/addons/base/test/__init__.py create mode 100644 openerp/addons/base/test/test_ir_cron.py diff --git a/openerp/addons/base/__init__.py b/openerp/addons/base/__init__.py index 847bef71f8c..16a61521fb4 100644 --- a/openerp/addons/base/__init__.py +++ b/openerp/addons/base/__init__.py @@ -24,6 +24,7 @@ import module import res import publisher_warranty import report +import test # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 84c9c5e837c..25393d50146 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -44,21 +44,6 @@ _intervalTypes = { 'minutes': lambda interval: relativedelta(minutes=interval), } -JOB = { - 'function': u'f', - 'interval_type': u'minutes', - 'user_id': 1, - 'name': u'test', - 'args': False, - 'numbercall': 1, - 'nextcall': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - 'priority': 5, - 'doall': True, - 'active': True, - 'interval_number': 1, - 'model': u'ir.cron' -} - class ir_cron(osv.osv): """ Model describing cron jobs (also called actions or tasks). """ @@ -93,19 +78,6 @@ class ir_cron(osv.osv): _logger = logging.getLogger('cron') - def f(a, b, c): - print ">>> in f" - - def expensive(a, b, c): - print ">>> in expensive" - time.sleep(80) - print ">>> out expensive" - - def expensive_2(a, b, c): - print ">>> in expensive_2" - time.sleep(30) - print ">>> out expensive_2" - def _check_args(self, cr, uid, ids, context=None): try: for this in self.browse(cr, uid, ids, context): @@ -287,44 +259,6 @@ class ir_cron(osv.osv): cr.commit() openerp.cron.schedule_in_advance(1, self.pool.db.dbname) - def _20_seconds(self, cr, uid): - print ">>> in _20_seconds" - time.sleep(20) - print ">>> out _20_seconds" - - def _80_seconds(self, cr, uid): - print ">>> in _80_seconds" - time.sleep(80) - print ">>> out _80_seconds" - - def test_0(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') - t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_0 _20_seconds A', function='_20_seconds', nextcall=t1)) - self.create(cr, uid, dict(JOB, name='test_0 _20_seconds B', function='_20_seconds', nextcall=t2)) - self.create(cr, uid, dict(JOB, name='test_0 _20_seconds C', function='_20_seconds', nextcall=t3)) - - def test_1(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_1 _20_seconds * 3', function='_20_seconds', nextcall=t1, numbercall=3)) - - def test_2(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_2 _80_seconds * 2', function='_80_seconds', nextcall=t1, numbercall=2)) - - def test_3(self, cr, uid): - now = datetime.now() - t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') - t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') - t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') - self.create(cr, uid, dict(JOB, name='test_3 _80_seconds A', function='_80_seconds', nextcall=t1)) - self.create(cr, uid, dict(JOB, name='test_3 _20_seconds B', function='_20_seconds', nextcall=t2)) - self.create(cr, uid, dict(JOB, name='test_3 _20_seconds C', function='_20_seconds', nextcall=t3)) - def create(self, cr, uid, vals, context=None): res = super(ir_cron, self).create(cr, uid, vals, context=context) self.update_running_cron(cr) @@ -342,4 +276,3 @@ class ir_cron(osv.osv): ir_cron() # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: - diff --git a/openerp/addons/base/test/__init__.py b/openerp/addons/base/test/__init__.py new file mode 100644 index 00000000000..2be9ed76bb4 --- /dev/null +++ b/openerp/addons/base/test/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011-TODAY OpenERP S.A. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +# Useful for manual testing of cron jobs scheduling. +# import test_ir_cron + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/test/test_ir_cron.py b/openerp/addons/base/test/test_ir_cron.py new file mode 100644 index 00000000000..2fe28e7c2ea --- /dev/null +++ b/openerp/addons/base/test/test_ir_cron.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011-TODAY OpenERP S.A. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +from datetime import datetime + +import openerp + +JOB = { + 'function': u'f', + 'interval_type': u'minutes', + 'user_id': 1, + 'name': u'test', + 'args': False, + 'numbercall': 1, + 'nextcall': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + 'priority': 5, + 'doall': True, + 'active': True, + 'interval_number': 1, + 'model': u'ir.cron' +} + +class test_ir_cron(openerp.osv.osv.osv): + """ Add a few handy methods to test cron jobs scheduling. """ + _inherit = "ir.cron" + + def _0_seconds(a, b, c): + print ">>> _0_seconds" + + def _20_seconds(self, cr, uid): + print ">>> in _20_seconds" + time.sleep(20) + print ">>> out _20_seconds" + + def _80_seconds(self, cr, uid): + print ">>> in _80_seconds" + time.sleep(80) + print ">>> out _80_seconds" + + def test_0(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') + t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_0 _20_seconds A', function='_20_seconds', nextcall=t1)) + self.create(cr, uid, dict(JOB, name='test_0 _20_seconds B', function='_20_seconds', nextcall=t2)) + self.create(cr, uid, dict(JOB, name='test_0 _20_seconds C', function='_20_seconds', nextcall=t3)) + + def test_1(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_1 _20_seconds * 3', function='_20_seconds', nextcall=t1, numbercall=3)) + + def test_2(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_2 _80_seconds * 2', function='_80_seconds', nextcall=t1, numbercall=2)) + + def test_3(self, cr, uid): + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') + t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_3 _80_seconds A', function='_80_seconds', nextcall=t1)) + self.create(cr, uid, dict(JOB, name='test_3 _20_seconds B', function='_20_seconds', nextcall=t2)) + self.create(cr, uid, dict(JOB, name='test_3 _20_seconds C', function='_20_seconds', nextcall=t3)) + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: + From ade53b64c7f69843e481262858935916bd922f47 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Mon, 8 Aug 2011 17:29:08 +0200 Subject: [PATCH 016/244] [IMP] cron: minor typo and comments. bzr revid: vmt@openerp.com-20110808152908-1v12zc7elj03mtnf --- openerp/addons/base/ir/ir_cron.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 25393d50146..fdfb6adc26a 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -178,7 +178,7 @@ class ir_cron(osv.osv): This selects in database all the jobs that should be processed. It then tries to lock each of them and, if it succeeds, spawns a thread to run the cron job (if it doesn't succeed, it means the job was already - locked to be taken care of by another thread. + locked to be taken care of by another thread). The cursor used to lock the job in database is given to the worker thread (which has to close it itself). @@ -206,11 +206,12 @@ class ir_cron(osv.osv): task_job = task_cr.dictfetchall()[0] except psycopg2.OperationalError, e: if e.pgcode == '55P03': - # Class 55: Object not in prerequisite state, 55P03: lock_not_available - # ... and fail. + # Class 55: Object not in prerequisite state; 55P03: lock_not_available + # ... and fail (in a good way for our purpose). print ">>>", job['name'], " is already being processed" continue else: + # ... and fail (badly). raise finally: if not task_job: From e93d018a394ed09c1e8420052b24c5c6c15dd568 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Tue, 9 Aug 2011 13:10:08 +0200 Subject: [PATCH 017/244] [IMP] cron: added yaml test. The test should probably be a standalone program as the YAML infrastructure isnt really suited for this purpose. bzr revid: vmt@openerp.com-20110809111008-vxh0bm08n3drw1o2 --- openerp/addons/base/__openerp__.py | 4 ++ openerp/addons/base/test/__init__.py | 2 + openerp/addons/base/test/test_ir_cron.py | 31 +++++++++++- openerp/addons/base/test/test_ir_cron.yml | 61 +++++++++++++++++++++++ openerp/cron.py | 35 +++++++------ openerp/modules/registry.py | 4 +- 6 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 openerp/addons/base/test/test_ir_cron.yml diff --git a/openerp/addons/base/__openerp__.py b/openerp/addons/base/__openerp__.py index 275b8317c0a..8e936a1566a 100644 --- a/openerp/addons/base/__openerp__.py +++ b/openerp/addons/base/__openerp__.py @@ -95,6 +95,10 @@ 'test/test_osv_expression.yml', 'test/test_ir_rule.yml', # <-- These tests modify/add/delete ir_rules. 'test/test_ir_values.yml', + # Commented because this takes some time. + # This must be (un)commented with the corresponding import statement + # in test/__init__.py. + # 'test/test_ir_cron.yml', # <-- These tests perform a roolback. ], 'installable': True, 'active': True, diff --git a/openerp/addons/base/test/__init__.py b/openerp/addons/base/test/__init__.py index 2be9ed76bb4..c0cc8f7cb78 100644 --- a/openerp/addons/base/test/__init__.py +++ b/openerp/addons/base/test/__init__.py @@ -20,6 +20,8 @@ ############################################################################## # Useful for manual testing of cron jobs scheduling. +# This must be (un)commented with the corresponding yml file +# in ../__openerp__.py. # import test_ir_cron # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/test/test_ir_cron.py b/openerp/addons/base/test/test_ir_cron.py index 2fe28e7c2ea..6ef79ac3a78 100644 --- a/openerp/addons/base/test/test_ir_cron.py +++ b/openerp/addons/base/test/test_ir_cron.py @@ -19,12 +19,14 @@ # ############################################################################## +import time from datetime import datetime +from dateutil.relativedelta import relativedelta import openerp JOB = { - 'function': u'f', + 'function': u'_0_seconds', 'interval_type': u'minutes', 'user_id': 1, 'name': u'test', @@ -83,5 +85,32 @@ class test_ir_cron(openerp.osv.osv.osv): self.create(cr, uid, dict(JOB, name='test_3 _20_seconds B', function='_20_seconds', nextcall=t2)) self.create(cr, uid, dict(JOB, name='test_3 _20_seconds C', function='_20_seconds', nextcall=t3)) + # This test assumes 4 cron threads. + def test_00(self, cr, uid): + self.test_00_set = set() + now = datetime.now() + t1 = (now + relativedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S') + t2 = (now + relativedelta(minutes=1, seconds=5)).strftime('%Y-%m-%d %H:%M:%S') + t3 = (now + relativedelta(minutes=1, seconds=10)).strftime('%Y-%m-%d %H:%M:%S') + self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_A', function='_20_seconds_A', nextcall=t1)) + self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_B', function='_20_seconds_B', nextcall=t2)) + self.create(cr, uid, dict(JOB, name='test_00 _20_seconds_C', function='_20_seconds_C', nextcall=t3)) + + def _expect(self, cr, uid, to_add, to_sleep, to_expect_in, to_expect_out): + assert self.test_00_set == to_expect_in + self.test_00_set.add(to_add) + time.sleep(to_sleep) + self.test_00_set.discard(to_add) + assert self.test_00_set == to_expect_out + + def _20_seconds_A(self, cr, uid): + self._expect(cr, uid, 'A', 20, set(), set(['B', 'C'])) + + def _20_seconds_B(self, cr, uid): + self._expect(cr, uid, 'B', 20, set('A'), set('C')) + + def _20_seconds_C(self, cr, uid): + self._expect(cr, uid, 'C', 20, set(['A', 'B']), set()) + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/test/test_ir_cron.yml b/openerp/addons/base/test/test_ir_cron.yml new file mode 100644 index 00000000000..5b6e36ae4ae --- /dev/null +++ b/openerp/addons/base/test/test_ir_cron.yml @@ -0,0 +1,61 @@ +- + Test the cron jobs scheduling. +- + Disable the existing cron jobs if any during the tests. +- + !python {model: ir.cron }: | + # For this test to work, as it involves multiple database cursors, + # we have to commit changes. But YAML tests must be rollbacked, so + # the final database state is left untouched. So we have to be a bit + # ugly here: use our own cursor, commit, and clean after ourselves. + # We also pass around some ids using setattr/delattr, and we have to + # rollback the previous tests otherwise we won't be able to touch the + # db. + # Well, this should probably be a standalone, or regular unit test, + # instead of using the YAML infrastructure. + cr.rollback() + our_cr = self.pool.db.cursor() + try: + ids = self.search(our_cr, uid, [], {}) + setattr(self, 'saved_ids', ids) + self.write(our_cr, uid, ids, {'active': False}, {}) + our_cr.commit() + finally: + our_cr.close() +- + Three concurrent jobs started with a slight time gap. Assume 4 cron threads. + This will take about 2 minutes. +- + !python {model: ir.cron }: | + # Pretend initialization is already done. We the use a try/finally + # to reset _init correctly. + self.pool._init = False + our_cr = self.pool.db.cursor() + try: + self.test_00(our_cr, uid) # this will commit using the passed cursor + import openerp.cron + openerp.cron._thread_count = 4 + # Wake up this db as soon as the master cron thread starts. + openerp.cron.schedule_in_advance(1, self.pool.db.dbname) + # Pretend to be the master thread, for 4 iterations. + openerp.cron.runner_body() + openerp.cron.runner_body() + openerp.cron.runner_body() + openerp.cron.runner_body() + finally: + self.pool._init = True + our_cr.close() +- + Clean after ourselves. +- + !python {model: ir.cron }: | + our_cr = self.pool.db.cursor() + try: + ids = [x for x in self.search(our_cr, uid, ['|', ('active', '=', True), ('active', '=', False)], {}) if x not in self.saved_ids] + self.unlink(our_cr, uid, ids, {}) + ids = self.saved_ids + delattr(self, 'saved_ids') + self.write(our_cr, uid, ids, {'active': True}, {}) + our_cr.commit() + finally: + our_cr.close() diff --git a/openerp/cron.py b/openerp/cron.py index 7f96a7d499d..64edf04208d 100644 --- a/openerp/cron.py +++ b/openerp/cron.py @@ -151,22 +151,25 @@ def runner(): checks every 60 seconds the next database wake-up. TODO: make configurable """ while True: - with _wakeups_lock: - while _wakeups and _wakeups[0][0] < time.time() and get_thread_count(): - task = heapq.heappop(_wakeups) - timestamp, db_name, canceled = task - if canceled: - continue - task[2] = True - registry = openerp.pooler.get_pool(db_name) - if not registry._init: - registry['ir.cron']._run_jobs() - amount = 60 - with _wakeups_lock: - # Sleep less than 60s if the next known wake-up will happen before. - if _wakeups and get_thread_count(): - amount = min(60, _wakeups[0][0] - time.time()) - time.sleep(amount) + runner_body() + +def runner_body(): + with _wakeups_lock: + while _wakeups and _wakeups[0][0] < time.time() and get_thread_count(): + task = heapq.heappop(_wakeups) + timestamp, db_name, canceled = task + if canceled: + continue + task[2] = True + registry = openerp.pooler.get_pool(db_name) + if not registry._init: + registry['ir.cron']._run_jobs() + amount = 60 + with _wakeups_lock: + # Sleep less than 60s if the next known wake-up will happen before. + if _wakeups and get_thread_count(): + amount = min(60, _wakeups[0][0] - time.time()) + time.sleep(amount) def start_master_thread(): diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 9ed51044793..ae6495dba5f 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -88,8 +88,8 @@ class Registry(object): """ Make the cron thread care about this registry/database jobs. This will initiate the cron thread to check for any pending jobs for - this registry/database as soon as possible. Then it will continously - monitors the ir.cron model for future jobs. See openerp.cron for + this registry/database as soon as possible. Then it will continuously + monitor the ir.cron model for future jobs. See openerp.cron for details. """ From bd03e6aabfe178476894912df1025a17253c6ddc Mon Sep 17 00:00:00 2001 From: "RavishchanraMurari (Open ERP)" Date: Thu, 15 Sep 2011 18:06:42 +0530 Subject: [PATCH 018/244] project_issue_kanban bzr revid: rmu@tinyerp.com-20110915123642-hiq0sfcunrciytk3 --- addons/project_issue/project_issue.py | 18 ++++- addons/project_issue/project_issue_menu.xml | 2 +- addons/project_issue/project_issue_view.xml | 75 +++++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/addons/project_issue/project_issue.py b/addons/project_issue/project_issue.py index 8237159af7a..128be8c2191 100644 --- a/addons/project_issue/project_issue.py +++ b/addons/project_issue/project_issue.py @@ -227,6 +227,7 @@ class project_issue(crm.crm_case, osv.osv): multi='compute_day', type="float", store=True), 'inactivity_days': fields.function(_compute_day, string='Days since last action', \ multi='compute_day', type="integer", help="Difference in days between last action and current date"), + 'color': fields.integer('Color Index'), 'message_ids': fields.one2many('mail.message', 'res_id', 'Messages', domain=[('model','=',_name)]), 'date_action_last': fields.datetime('Last Action', readonly=1), 'date_action_next': fields.datetime('Next Action', readonly=1), @@ -268,7 +269,22 @@ class project_issue(crm.crm_case, osv.osv): 'project_id':_get_project, 'categ_id' : lambda *a: False, #'assigned_to' : lambda obj, cr, uid, context: uid, - } + } + + def set_priority(self, cr, uid, ids, priority): + """Set lead priority + """ + return self.write(cr, uid, ids, {'priority' : priority}) + + def set_high_priority(self, cr, uid, ids, *args): + """Set lead priority to high + """ + return self.set_priority(cr, uid, ids, '1') + + def set_normal_priority(self, cr, uid, ids, *args): + """Set lead priority to normal + """ + return self.set_priority(cr, uid, ids, '3') def convert_issue_task(self, cr, uid, ids, context=None): case_obj = self.pool.get('project.issue') diff --git a/addons/project_issue/project_issue_menu.xml b/addons/project_issue/project_issue_menu.xml index 18c8779704f..0915b2b96d7 100644 --- a/addons/project_issue/project_issue_menu.xml +++ b/addons/project_issue/project_issue_menu.xml @@ -10,7 +10,7 @@ Issues project.issue form - tree,calendar + tree,calendar,kanban {"search_default_user_id": uid, "search_default_current":1, "search_default_project_id":project_id} diff --git a/addons/project_issue/project_issue_view.xml b/addons/project_issue/project_issue_view.xml index 0bf171d6aeb..b54690a3deb 100644 --- a/addons/project_issue/project_issue_view.xml +++ b/addons/project_issue/project_issue_view.xml @@ -263,6 +263,81 @@ + + + + + Project Issue Kanban + project.issue + kanban + + + + + + + + + + + + +
+
+ + + + + + +
+ + +
+ +
+
+ +
+
+ +
+
+ +
+
+ Version +
+
+ +
+
+ + + + + +
+
+ + + +
+
+
+
+
+ + + + + + + + + + + # ------------------------------------------------------ # Feature Requests # ------------------------------------------------------ From d7e38b95b498b8f1c8028327a8dbd636cddbd0d0 Mon Sep 17 00:00:00 2001 From: "RavishchanraMurari (Open ERP)" Date: Fri, 16 Sep 2011 12:05:55 +0530 Subject: [PATCH 019/244] project_issue_karban_view bzr revid: rmu@tinyerp.com-20110916063555-m9l28fb9ravuf7r9 --- addons/project_issue/project_issue_view.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/addons/project_issue/project_issue_view.xml b/addons/project_issue/project_issue_view.xml index b54690a3deb..9c10b958d80 100644 --- a/addons/project_issue/project_issue_view.xml +++ b/addons/project_issue/project_issue_view.xml @@ -284,13 +284,13 @@
- - - +
+ + +
-
@@ -306,7 +306,7 @@
- Version + Version
From dc1ba50c5e1f857e869b1f6aa508f7517bc8a4a6 Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Fri, 16 Sep 2011 15:42:14 +0530 Subject: [PATCH 020/244] [IMP]:hr:added kanban view for hr employess. bzr revid: apa@tinyerp.com-20110916101214-hlff8uqv8y87h5bi --- addons/hr/__openerp__.py | 2 +- addons/hr/hr.py | 5 ++- addons/hr/hr_view.xml | 83 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/addons/hr/__openerp__.py b/addons/hr/__openerp__.py index 598b49a725e..bca65c54cb2 100644 --- a/addons/hr/__openerp__.py +++ b/addons/hr/__openerp__.py @@ -38,7 +38,7 @@ You can manage: 'author': 'OpenERP SA', 'website': 'http://www.openerp.com', 'images': ['images/hr_department.jpeg', 'images/hr_employee.jpeg','images/hr_job_position.jpeg'], - 'depends': ['base_setup', 'resource', 'board'], + 'depends': ['base_setup','mail', 'resource', 'board'], 'init_xml': [], 'update_xml': [ 'security/hr_security.xml', diff --git a/addons/hr/hr.py b/addons/hr/hr.py index 1424d5e3378..512985b9b32 100644 --- a/addons/hr/hr.py +++ b/addons/hr/hr.py @@ -149,7 +149,9 @@ class hr_employee(osv.osv): 'coach_id': fields.many2one('hr.employee', 'Coach'), 'job_id': fields.many2one('hr.job', 'Job'), 'photo': fields.binary('Photo'), - 'passport_id':fields.char('Passport No', size=64) + 'passport_id':fields.char('Passport No', size=64), + 'color': fields.integer('Color Index'), + 'city': fields.related('address_id', 'city', type='char', string='City'), } def unlink(self, cr, uid, ids, context=None): @@ -198,6 +200,7 @@ class hr_employee(osv.osv): 'active': 1, 'photo': _get_photo, 'marital': 'single', + 'color': 0, } def _check_recursion(self, cr, uid, ids, context=None): diff --git a/addons/hr/hr_view.xml b/addons/hr/hr_view.xml index fb10baedc9e..d71f78a3053 100644 --- a/addons/hr/hr_view.xml +++ b/addons/hr/hr_view.xml @@ -125,7 +125,72 @@ - + + + HR - Employess Kanban + hr.employee + kanban + + + + + + + + + + + + + + + + + + Employees Structure hr.employee @@ -149,13 +214,27 @@ Employees hr.employee form - tree,form + tree,form,kanban [] {"search_default_department_id": department_id,"search_default_active":eval('True')} Here you can manage your work force by creating employees and assigning them specific properties in the system. Maintain all employee related information and keep track of anything that needs to be recorded for them. The personal information tab will help you maintain their identity data. The Categories tab gives you the opportunity to assign them related employee categories depending on their position and activities within the company. A category can be a seniority level within the company or a department. The Timesheets tab allows to assign them a specific timesheet and analytic journal where they will be able to enter time through the system. In the note tab, you can enter text data that should be recorded for a specific employee. + + + + tree + + + + + + + form + + + From fce6f5932b1859db8e26dffff5fbb0d445885df5 Mon Sep 17 00:00:00 2001 From: "Tejas (OpenERP)" Date: Fri, 16 Sep 2011 16:15:01 +0530 Subject: [PATCH 021/244] [kanban] contacts view for kanban bzr revid: tta@openerp.com-20110916104501-tuwftefbl7lumix7 --- openerp/addons/base/res/res_partner.py | 1 + openerp/addons/base/res/res_partner_view.xml | 64 ++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/openerp/addons/base/res/res_partner.py b/openerp/addons/base/res/res_partner.py index 4b8f1279787..2e692e2d4b7 100644 --- a/openerp/addons/base/res/res_partner.py +++ b/openerp/addons/base/res/res_partner.py @@ -300,6 +300,7 @@ class res_partner_address(osv.osv): 'active': fields.boolean('Active', help="Uncheck the active field to hide the contact."), # 'company_id': fields.related('partner_id','company_id',type='many2one',relation='res.company',string='Company', store=True), 'company_id': fields.many2one('res.company', 'Company',select=1), + 'color': fields.integer('Color Index'), } _defaults = { 'active': lambda *a: 1, diff --git a/openerp/addons/base/res/res_partner_view.xml b/openerp/addons/base/res/res_partner_view.xml index 8b26023701d..c47edc57aad 100644 --- a/openerp/addons/base/res/res_partner_view.xml +++ b/openerp/addons/base/res/res_partner_view.xml @@ -105,11 +105,75 @@ + + + res.partner.address.kanban + res.partner.address + kanban + + + + + + + + + + + + + + + + + + + + Addresses ir.actions.act_window res.partner.address form + tree,form,kanban {"search_default_customer":1} Customers (also called Partners in other areas of the system) helps you manage your address book of companies whether they are prospects, customers and/or suppliers. The partner form allows you to track and record all the necessary information to interact with your partners from the company address to their contacts as well as pricelists, and much more. If you installed the CRM, with the history tab, you can track all the interactions with a partner such as opportunities, emails, or sales orders issued. From 00a2a259301fe612f8d7247563ff956a3b5f929e Mon Sep 17 00:00:00 2001 From: "Tejas (OpenERP)" Date: Fri, 16 Sep 2011 16:45:26 +0530 Subject: [PATCH 022/244] [kanban] id changed and make email and phone italic bzr revid: tta@openerp.com-20110916111526-znb3mc918jfzosw7 --- openerp/addons/base/res/res_partner_view.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openerp/addons/base/res/res_partner_view.xml b/openerp/addons/base/res/res_partner_view.xml index c47edc57aad..35c31cb0b06 100644 --- a/openerp/addons/base/res/res_partner_view.xml +++ b/openerp/addons/base/res/res_partner_view.xml @@ -106,7 +106,7 @@ - + res.partner.address.kanban res.partner.address kanban @@ -143,9 +143,9 @@ ,
,
- + , - + From 2e35f396065201547bb289a458e9e51e6d343798 Mon Sep 17 00:00:00 2001 From: "Tejas (OpenERP)" Date: Fri, 16 Sep 2011 17:19:45 +0530 Subject: [PATCH 023/244] [kanban] bold tag removed bzr revid: tta@openerp.com-20110916114945-3zge1h6e6x2ohxnm --- openerp/addons/base/res/res_partner_view.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openerp/addons/base/res/res_partner_view.xml b/openerp/addons/base/res/res_partner_view.xml index 35c31cb0b06..722d11c0770 100644 --- a/openerp/addons/base/res/res_partner_view.xml +++ b/openerp/addons/base/res/res_partner_view.xml @@ -106,7 +106,7 @@ - + res.partner.address.kanban res.partner.address kanban @@ -142,10 +142,10 @@ ,
- ,
- + ,
+ , - + From 7123f18cc0eb19c36b9cc08d3755864978ffa215 Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Fri, 16 Sep 2011 17:33:52 +0530 Subject: [PATCH 024/244] [IMP]:applied amb's patch and improved it for add the kanban view for hr_recruitment. bzr revid: apa@tinyerp.com-20110916120352-4o5ycwzx3p9zktax --- addons/hr_recruitment/hr_recruitment.py | 17 ++++ addons/hr_recruitment/hr_recruitment_menu.xml | 2 +- addons/hr_recruitment/hr_recruitment_view.xml | 79 +++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/addons/hr_recruitment/hr_recruitment.py b/addons/hr_recruitment/hr_recruitment.py index 7fde90dc8e9..1499243500b 100644 --- a/addons/hr_recruitment/hr_recruitment.py +++ b/addons/hr_recruitment/hr_recruitment.py @@ -177,6 +177,7 @@ class hr_applicant(crm.crm_case, osv.osv): multi='day_open', type="float", store=True), 'day_close': fields.function(_compute_day, string='Days to Close', \ multi='day_close', type="float", store=True), + 'color': fields.integer('Color Index'), } def _get_stage(self, cr, uid, context=None): @@ -191,6 +192,7 @@ class hr_applicant(crm.crm_case, osv.osv): 'priority': lambda *a: '', 'company_id': lambda s, cr, uid, c: s.pool.get('res.company')._company_default_get(cr, uid, 'crm.helpdesk', context=c), 'priority': lambda *a: crm.AVAILABLE_PRIORITIES[2][0], + 'color': 0, } def onchange_job(self,cr, uid, ids, job, context=None): @@ -440,6 +442,21 @@ class hr_applicant(crm.crm_case, osv.osv): res = super(hr_applicant, self).case_reset(cr, uid, ids, *args) self.write(cr, uid, ids, {'date_open': False, 'date_closed': False}) return res + + def set_priority(self, cr, uid, ids, priority): + """Set lead priority + """ + return self.write(cr, uid, ids, {'priority' : priority}) + + def set_high_priority(self, cr, uid, ids, *args): + """Set lead priority to high + """ + return self.set_priority(cr, uid, ids, '1') + + def set_normal_priority(self, cr, uid, ids, *args): + """Set lead priority to normal + """ + return self.set_priority(cr, uid, ids, '3') hr_applicant() diff --git a/addons/hr_recruitment/hr_recruitment_menu.xml b/addons/hr_recruitment/hr_recruitment_menu.xml index 9883e311fba..0bac3f8615a 100644 --- a/addons/hr_recruitment/hr_recruitment_menu.xml +++ b/addons/hr_recruitment/hr_recruitment_menu.xml @@ -6,7 +6,7 @@ Applicants hr.applicant - tree,form,graph,calendar + tree,form,graph,calendar,kanban {"search_default_user_id":uid, 'search_default_current': 1,"search_default_department_id": department_id} diff --git a/addons/hr_recruitment/hr_recruitment_view.xml b/addons/hr_recruitment/hr_recruitment_view.xml index 629b1c68012..290dedd2fb3 100644 --- a/addons/hr_recruitment/hr_recruitment_view.xml +++ b/addons/hr_recruitment/hr_recruitment_view.xml @@ -283,7 +283,86 @@ + + + Hr Applicants kanban + hr.applicant + kanban + + + + + + + + + + +
+
+
+ + + + + +
+ + + + + +
+
+
+ + + + +
+
+ +
+
+ + + (Not Good) + (On Average) + (Good) + (Very Good) + (Excellent) +
+
+ +
+
+ +
+
+
+
+
+ + + + + + +
+
+ + +
+
+
+
+
+ + + + + # ------------------------------------------------------ # HR Job # ------------------------------------------------------ From dc54be9bb925f50008907dc657a8a5e8bcb2fcbe Mon Sep 17 00:00:00 2001 From: "Amit Bhavsar (Open ERP)" Date: Fri, 16 Sep 2011 18:17:25 +0530 Subject: [PATCH 025/244] [kabanization],kanban view for the partner bzr revid: amb@tinyerp.com-20110916124725-1d07ph63gx8nv3u1 --- openerp/addons/base/res/res_partner.py | 4 ++ openerp/addons/base/res/res_partner_view.xml | 72 ++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/openerp/addons/base/res/res_partner.py b/openerp/addons/base/res/res_partner.py index 4b8f1279787..d88bf8463a4 100644 --- a/openerp/addons/base/res/res_partner.py +++ b/openerp/addons/base/res/res_partner.py @@ -131,12 +131,15 @@ class res_partner(osv.osv): 'customer': fields.boolean('Customer', help="Check this box if the partner is a customer."), 'supplier': fields.boolean('Supplier', help="Check this box if the partner is a supplier. If it's not checked, purchase people will not see it when encoding a purchase order."), 'city': fields.related('address', 'city', type='char', string='City'), + 'function': fields.related('address', 'function', type='char', string='function'), + 'subname': fields.related('address', 'name', type='char', string='Contact Name'), 'phone': fields.related('address', 'phone', type='char', string='Phone'), 'mobile': fields.related('address', 'mobile', type='char', string='Mobile'), 'country': fields.related('address', 'country_id', type='many2one', relation='res.country', string='Country'), 'employee': fields.boolean('Employee', help="Check this box if the partner is an Employee."), 'email': fields.related('address', 'email', type='char', size=240, string='E-mail'), 'company_id': fields.many2one('res.company', 'Company', select=1), + 'color': fields.integer('Color Index'), } def _default_category(self, cr, uid, context={}): @@ -150,6 +153,7 @@ class res_partner(osv.osv): 'address': [{'type': 'default'}], 'category_id': _default_category, 'company_id': lambda s,cr,uid,c: s.pool.get('res.company')._company_default_get(cr, uid, 'res.partner', context=c), + 'color': 0, } def copy(self, cr, uid, id, default={}, context={}): diff --git a/openerp/addons/base/res/res_partner_view.xml b/openerp/addons/base/res/res_partner_view.xml index 8b26023701d..8f7111bd738 100644 --- a/openerp/addons/base/res/res_partner_view.xml +++ b/openerp/addons/base/res/res_partner_view.xml @@ -353,12 +353,84 @@ + + + + RES - PARTNER KANBAN + res.partner + kanban + + + + + + + + + + + + + + + + + + + Customers ir.actions.act_window res.partner form + kanban {"search_default_customer":1} A customer is an entity you do business with, like a company or an organization. A customer can have several contacts or addresses which are the people working for this company. You can use the history tab, to follow all transactions related to a customer: sales order, emails, opportunities, claims, etc. If you use the email gateway, the Outlook or the Thunderbird plugin, don't forget to register emails to each contact so that the gateway will automatically attach incoming emails to the right partner. From e5e20ba47cfb68fb23d33b7a911f7ca5cb92108a Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Fri, 16 Sep 2011 18:43:02 +0530 Subject: [PATCH 026/244] [IMP]:improved bzr revid: apa@tinyerp.com-20110916131302-3jjj3ly3xy00f0so --- addons/project_issue/project_issue_view.xml | 93 +++++++++++---------- 1 file changed, 51 insertions(+), 42 deletions(-) diff --git a/addons/project_issue/project_issue_view.xml b/addons/project_issue/project_issue_view.xml index 9c10b958d80..605c26f232b 100644 --- a/addons/project_issue/project_issue_view.xml +++ b/addons/project_issue/project_issue_view.xml @@ -274,58 +274,67 @@ - - - - - - - + + + + + + +
- - - - - -
- - - -
- -
-
- -
-
- -
-
- -
-
- Version -
-
- -
-
+
+ + + + + +
+ + + + + +
+
+
+ + + + +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+
- - -
-
- + + + +
+
+

-
-
+
+
From 0ee38e29ade5acc7fd9aa8955901c5fbc69c788f Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Fri, 16 Sep 2011 15:22:28 +0200 Subject: [PATCH 027/244] [IMP] convert form widgets to use a root class to find themselves back in the rendered dom, instead of a root id bzr revid: xmo@openerp.com-20110916132228-h2qszo33q6z289qf --- addons/web/static/src/js/view_form.js | 22 ++++++++-------- addons/web/static/src/xml/base.xml | 37 +++++++++++++-------------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/addons/web/static/src/js/view_form.js b/addons/web/static/src/js/view_form.js index e28936b850d..16a4ba908a7 100644 --- a/addons/web/static/src/js/view_form.js +++ b/addons/web/static/src/js/view_form.js @@ -82,7 +82,7 @@ openerp.web.FormView = openerp.web.View.extend( /** @lends openerp.web.FormView# _.each(this.widgets, function(w) { w.start(); }); - this.$form_header = this.$element.find('#' + this.element_id + '_header'); + this.$form_header = this.$element.find('.oe_form_header'); this.$form_header.find('div.oe_form_pager button[data-pager-action]').click(function() { var action = $(this).data('pager-action'); self.on_pager_action(action); @@ -196,7 +196,7 @@ openerp.web.FormView = openerp.web.View.extend( /** @lends openerp.web.FormView# } }, do_update_pager: function(hide_index) { - var $pager = this.$element.find('#' + this.element_id + '_header div.oe_form_pager'); + var $pager = this.$form_header.find('div.oe_form_pager'); var index = hide_index ? '-' : this.dataset.index + 1; $pager.find('span.oe_pager_index').html(index); $pager.find('span.oe_pager_count').html(this.dataset.ids.length); @@ -619,6 +619,7 @@ openerp.web.form.compute_domain = function(expr, fields) { openerp.web.form.Widget = openerp.web.Widget.extend(/** @lends openerp.web.form.Widget# */{ template: 'Widget', + identifier_prefix: 'formview-widget-', /** * @constructs openerp.web.form.Widget * @extends openerp.web.Widget @@ -632,11 +633,13 @@ openerp.web.form.Widget = openerp.web.Widget.extend(/** @lends openerp.web.form. this.modifiers = JSON.parse(this.node.attrs.modifiers || '{}'); this.type = this.type || node.tag; this.element_name = this.element_name || this.type; - this.element_id = [this.view.element_id, this.element_name, this.view.widgets_counter++].join("_"); + this.element_class = [ + 'formview', this.view.view_id, this.element_name, + this.view.widgets_counter++].join("_"); - this._super(view, this.element_id); + this._super(view); - this.view.widgets[this.element_id] = this; + this.view.widgets[this.element_class] = this; this.children = node.children; this.colspan = parseInt(node.attrs.colspan || 1, 10); this.decrease_max_width = 0; @@ -649,7 +652,7 @@ openerp.web.form.Widget = openerp.web.Widget.extend(/** @lends openerp.web.form. this.width = this.node.attrs.width; }, start: function() { - this.$element = $('#' + this.element_id); + this.$element = this.view.$element.find('.' + this.element_class); }, stop: function() { if (this.$element) { @@ -896,7 +899,7 @@ openerp.web.form.WidgetLabel = openerp.web.form.Widget.extend({ var self = this; this.$element.find("label").dblclick(function() { var widget = self['for'] || self; - console.log(widget.element_id , widget); + console.log(widget.element_class , widget); window.w = widget; }); } @@ -1932,9 +1935,8 @@ openerp.web.form.FieldOne2Many = openerp.web.form.Field.extend({ }, validate: function() { this.invalid = false; - var self = this; - var view = self.viewmanager.views[self.viewmanager.active_view].controller; - if (self.viewmanager.active_view === "form") { + var view = this.viewmanager.views[this.viewmanager.active_view].controller; + if (this.viewmanager.active_view === "form") { for (var f in view.fields) { f = view.fields[f]; if (!f.is_valid()) { diff --git a/addons/web/static/src/xml/base.xml b/addons/web/static/src/xml/base.xml index c2b6ca0bdf2..daf85c43ee7 100644 --- a/addons/web/static/src/xml/base.xml +++ b/addons/web/static/src/xml/base.xml @@ -649,7 +649,7 @@ -
+
-
-
- - , - -
-
- - , - -
-
- - - , - - -
-
- - -
-
- - - +
+
+ + + + + +
+ + +
+ + , + +
+
+ + , + +
+
+ + , + +
+
+
+
+
+ + + - -
-
+
- +
- - - - - +
+ + + + + Customers From ad60a40879c05116fd9a4998e49fa1939569e21d Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Mon, 19 Sep 2011 14:47:01 +0530 Subject: [PATCH 033/244] [IMP]:improved bzr revid: apa@tinyerp.com-20110919091701-t4jzmqm4245knacb --- openerp/addons/base/res/res_partner_view.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/res/res_partner_view.xml b/openerp/addons/base/res/res_partner_view.xml index 221d5c34119..efac1a6c603 100644 --- a/openerp/addons/base/res/res_partner_view.xml +++ b/openerp/addons/base/res/res_partner_view.xml @@ -390,7 +390,7 @@
- +
From c8c52f6257fc2157ef979bf89435cb289dfb6644 Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Mon, 19 Sep 2011 15:03:43 +0530 Subject: [PATCH 034/244] improved bzr revid: apa@tinyerp.com-20110919093343-nykuuvw7d8d0o1xn --- addons/hr/hr_view.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/hr/hr_view.xml b/addons/hr/hr_view.xml index d71f78a3053..4b5d64746a9 100644 --- a/addons/hr/hr_view.xml +++ b/addons/hr/hr_view.xml @@ -157,8 +157,8 @@
-
- + +
From b4e8ac180c6811587e036c62a6c0603fadd2277c Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 19 Sep 2011 12:15:17 +0200 Subject: [PATCH 035/244] [REM] redundant initializers in formview widgets: move most formview widget templates to the class-level bzr revid: xmo@openerp.com-20110919101517-40303tgi58cfshtk --- addons/web/static/src/js/view_form.js | 58 ++++++++------------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/addons/web/static/src/js/view_form.js b/addons/web/static/src/js/view_form.js index b1ce7542210..23a0f19f121 100644 --- a/addons/web/static/src/js/view_form.js +++ b/addons/web/static/src/js/view_form.js @@ -816,9 +816,9 @@ openerp.web.form.WidgetNotebookPage = openerp.web.form.WidgetFrame.extend({ }); openerp.web.form.WidgetSeparator = openerp.web.form.Widget.extend({ + template: 'WidgetSeparator', init: function(view, node) { this._super(view, node); - this.template = "WidgetSeparator"; this.orientation = node.attrs.orientation || 'horizontal'; if (this.orientation === 'vertical') { this.width = '1'; @@ -828,9 +828,9 @@ openerp.web.form.WidgetSeparator = openerp.web.form.Widget.extend({ }); openerp.web.form.WidgetButton = openerp.web.form.Widget.extend({ + template: 'WidgetButton', init: function(view, node) { this._super(view, node); - this.template = "WidgetButton"; if (this.string) { // We don't have button key bindings in the webclient this.string = this.string.replace(/_/g, ''); @@ -881,6 +881,7 @@ openerp.web.form.WidgetButton = openerp.web.form.Widget.extend({ }); openerp.web.form.WidgetLabel = openerp.web.form.Widget.extend({ + template: 'WidgetLabel', init: function(view, node) { this.element_name = 'label_' + node.attrs.name; @@ -891,7 +892,6 @@ openerp.web.form.WidgetLabel = openerp.web.form.Widget.extend({ this.template = "WidgetParagraph"; this.colspan = parseInt(this.node.attrs.colspan || 1, 10); } else { - this.template = "WidgetLabel"; this.colspan = 1; this.width = '1%'; this.decrease_max_width = 1; @@ -1049,10 +1049,7 @@ openerp.web.form.Field = openerp.web.form.Widget.extend(/** @lends openerp.web.f }); openerp.web.form.FieldChar = openerp.web.form.Field.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldChar"; - }, + template: 'FieldChar', start: function() { this._super.apply(this, arguments); this.$element.find('input').change(this.on_ui_change); @@ -1085,10 +1082,7 @@ openerp.web.form.FieldChar = openerp.web.form.Field.extend({ }); openerp.web.form.FieldEmail = openerp.web.form.FieldChar.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldEmail"; - }, + template: 'FieldEmail', start: function() { this._super.apply(this, arguments); this.$element.find('button').click(this.on_button_clicked); @@ -1107,10 +1101,7 @@ openerp.web.form.FieldEmail = openerp.web.form.FieldChar.extend({ }); openerp.web.form.FieldUrl = openerp.web.form.FieldChar.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldUrl"; - }, + template: 'FieldUrl', start: function() { this._super.apply(this, arguments); this.$element.find('button').click(this.on_button_clicked); @@ -1136,9 +1127,9 @@ openerp.web.form.FieldFloat = openerp.web.form.FieldChar.extend({ }); openerp.web.form.FieldDatetime = openerp.web.form.Field.extend({ + template: 'FieldDate', init: function(view, node) { this._super(view, node); - this.template = "FieldDate"; this.jqueryui_object = 'datetimepicker'; }, start: function() { @@ -1229,10 +1220,7 @@ openerp.web.form.FieldDate = openerp.web.form.FieldDatetime.extend({ }); openerp.web.form.FieldText = openerp.web.form.Field.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldText"; - }, + template: 'FieldText', start: function() { this._super.apply(this, arguments); this.$element.find('textarea').change(this.on_ui_change); @@ -1265,10 +1253,7 @@ openerp.web.form.FieldText = openerp.web.form.Field.extend({ }); openerp.web.form.FieldBoolean = openerp.web.form.Field.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldBoolean"; - }, + template: 'FieldBoolean', start: function() { var self = this; this._super.apply(this, arguments); @@ -1299,10 +1284,7 @@ openerp.web.form.FieldBoolean = openerp.web.form.Field.extend({ }); openerp.web.form.FieldProgressBar = openerp.web.form.Field.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldProgressBar"; - }, + template: 'FieldProgressBar', start: function() { this._super.apply(this, arguments); this.$element.find('div').progressbar({ @@ -1325,10 +1307,10 @@ openerp.web.form.FieldTextXml = openerp.web.form.Field.extend({ }); openerp.web.form.FieldSelection = openerp.web.form.Field.extend({ + template: 'FieldSelection', init: function(view, node) { var self = this; this._super(view, node); - this.template = "FieldSelection"; this.values = this.field.selection; _.each(this.values, function(v, i) { if (v[0] === false && v[1] === '') { @@ -1435,9 +1417,9 @@ openerp.web.form.dialog = function(content, options) { }; openerp.web.form.FieldMany2One = openerp.web.form.Field.extend({ + template: 'FieldMany2One', init: function(view, node) { this._super(view, node); - this.template = "FieldMany2One"; this.limit = 7; this.value = null; this.cm_id = _.uniqueId('m2o_cm_'); @@ -1762,10 +1744,10 @@ var commands = { } }; openerp.web.form.FieldOne2Many = openerp.web.form.Field.extend({ + template: 'FieldOne2Many', multi_selection: false, init: function(view, node) { this._super(view, node); - this.template = "FieldOne2Many"; this.is_started = $.Deferred(); this.form_last_update = $.Deferred(); this.disable_utility_classes = true; @@ -2020,10 +2002,10 @@ openerp.web.form.One2ManyListView = openerp.web.ListView.extend({ }); openerp.web.form.FieldMany2Many = openerp.web.form.Field.extend({ + template: 'FieldMany2Many', multi_selection: false, init: function(view, node) { this._super(view, node); - this.template = "FieldMany2Many"; this.list_id = _.uniqueId("many2many"); this.is_started = $.Deferred(); }, @@ -2365,9 +2347,9 @@ openerp.web.form.FormOpenDataset = openerp.web.ReadOnlyDataSetSearch.extend({ }); openerp.web.form.FieldReference = openerp.web.form.Field.extend({ + template: 'FieldReference', init: function(view, node) { this._super(view, node); - this.template = "FieldReference"; this.fields_view = { fields: { selection: { @@ -2502,10 +2484,7 @@ openerp.web.form.FieldBinary = openerp.web.form.Field.extend({ }); openerp.web.form.FieldBinaryFile = openerp.web.form.FieldBinary.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldBinaryFile"; - }, + template: 'FieldBinaryFile', set_value: function(value) { this._super.apply(this, arguments); var show_value = (value != null && value !== false) ? value : ''; @@ -2533,10 +2512,7 @@ openerp.web.form.FieldBinaryFile = openerp.web.form.FieldBinary.extend({ }); openerp.web.form.FieldBinaryImage = openerp.web.form.FieldBinary.extend({ - init: function(view, node) { - this._super(view, node); - this.template = "FieldBinaryImage"; - }, + template: 'FieldBinaryImage', start: function() { this._super.apply(this, arguments); this.$image = this.$element.find('img.oe-binary-image'); From 04f843a986181a74d5512c9916f153faf8fb5080 Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Mon, 19 Sep 2011 15:54:16 +0530 Subject: [PATCH 036/244] set images bzr revid: apa@tinyerp.com-20110919102416-x0vk0x60llm1qjjb --- addons/hr/hr_view.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/hr/hr_view.xml b/addons/hr/hr_view.xml index 4b5d64746a9..7b7b25a1557 100644 --- a/addons/hr/hr_view.xml +++ b/addons/hr/hr_view.xml @@ -158,7 +158,7 @@ -
- +
From 2ad93ea0cd777538670dcab06c995586b14dbfc7 Mon Sep 17 00:00:00 2001 From: "Amit (OpenERP)" Date: Mon, 19 Sep 2011 16:17:42 +0530 Subject: [PATCH 037/244] [IMP] bzr revid: apa@tinyerp.com-20110919104742-f52pkqpluov2aob6 --- addons/hr/hr_view.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/hr/hr_view.xml b/addons/hr/hr_view.xml index 7b7b25a1557..b5192b63d44 100644 --- a/addons/hr/hr_view.xml +++ b/addons/hr/hr_view.xml @@ -158,7 +158,7 @@
- +
From ea6299dae6e1a5341e6bec8d164c642492af0d12 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 19 Sep 2011 13:57:46 +0200 Subject: [PATCH 038/244] [FIX] form.FieldOne2Many, nesting itself inside itself, breaking the self-matcher bzr revid: xmo@openerp.com-20110919115746-16jz8hajcdgl010f --- addons/web/static/src/xml/base.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/addons/web/static/src/xml/base.xml b/addons/web/static/src/xml/base.xml index 498a8bb1dd2..c88b0116a27 100644 --- a/addons/web/static/src/xml/base.xml +++ b/addons/web/static/src/xml/base.xml @@ -855,8 +855,6 @@ -
-
From 566f9a43e8fce80cb53386aa4a003f4c6af21113 Mon Sep 17 00:00:00 2001 From: Xavier Morel Date: Mon, 19 Sep 2011 14:24:16 +0200 Subject: [PATCH 039/244] [ADD] readonly branch to a bunch of form widgets bzr revid: xmo@openerp.com-20110919122416-h773tyz1kogazyy0 --- addons/web/static/src/js/view_form.js | 70 +++++++++++++++++++---- addons/web/static/src/xml/base.xml | 80 +++++++++++++++++++++------ 2 files changed, 122 insertions(+), 28 deletions(-) diff --git a/addons/web/static/src/js/view_form.js b/addons/web/static/src/js/view_form.js index 23a0f19f121..1a14cf1745d 100644 --- a/addons/web/static/src/js/view_form.js +++ b/addons/web/static/src/js/view_form.js @@ -1057,7 +1057,12 @@ openerp.web.form.FieldChar = openerp.web.form.Field.extend({ set_value: function(value) { this._super.apply(this, arguments); var show_value = openerp.web.format_value(value, this, ''); - this.$element.find('input').val(show_value); + if (this.view.readonly) { + this.$element.find('div').text(show_value); + } else { + this.$element.find('input').val(show_value); + } + return show_value; }, update_dom: function() { this._super.apply(this, arguments); @@ -1095,8 +1100,12 @@ openerp.web.form.FieldEmail = openerp.web.form.FieldChar.extend({ } }, set_value: function(value) { - this._super.apply(this, arguments); - this.$element.find('a').attr('href', 'mailto:' + this.$element.find('input').val()); + var displayed = this._super.apply(this, arguments); + if (this.view.readonly) { + this.$element.find('a') + .attr('href', 'mailto:' + displayed) + .text(displayed); + } } }); @@ -1112,6 +1121,14 @@ openerp.web.form.FieldUrl = openerp.web.form.FieldChar.extend({ } else { window.open(this.value); } + }, + set_value: function(value) { + var displayed = this._super.apply(this, arguments); + if (this.view.readonly) { + this.$element.find('a') + .attr('href', displayed) + .text(displayed); + } } }); @@ -1130,11 +1147,14 @@ openerp.web.form.FieldDatetime = openerp.web.form.Field.extend({ template: 'FieldDate', init: function(view, node) { this._super(view, node); - this.jqueryui_object = 'datetimepicker'; + if (!this.view.readonly) { + this.jqueryui_object = 'datetimepicker'; + } }, start: function() { var self = this; this._super.apply(this, arguments); + if (this.view.readonly) { return; } this.$element.find('input').change(this.on_ui_change); this.picker({ onSelect: this.on_picker_select, @@ -1164,7 +1184,11 @@ openerp.web.form.FieldDatetime = openerp.web.form.Field.extend({ set_value: function(value) { value = this.parse(value); this._super(value); - this.$element.find('input').val(value ? this.format_client(value) : ''); + if (this.view.readonly) { + this.$element.find('div').text(value ? this.format_client(value) : ''); + } else { + this.$element.find('input').val(value ? this.format_client(value) : ''); + } }, get_value: function() { return this.format(this.value); @@ -1211,7 +1235,9 @@ openerp.web.form.FieldDatetime = openerp.web.form.Field.extend({ openerp.web.form.FieldDate = openerp.web.form.FieldDatetime.extend({ init: function(view, node) { this._super(view, node); - this.jqueryui_object = 'datepicker'; + if (!this.view.readonly) { + this.jqueryui_object = 'datepicker'; + } }, on_picker_select: function(text, instance) { this._super(text, instance); @@ -1228,7 +1254,11 @@ openerp.web.form.FieldText = openerp.web.form.Field.extend({ set_value: function(value) { this._super.apply(this, arguments); var show_value = openerp.web.format_value(value, this, ''); - this.$element.find('textarea').val(show_value); + if (this.view.readonly) { + this.$element.find('div').text(show_value); + } else { + this.$element.find('textarea').val(show_value); + } }, update_dom: function() { this._super.apply(this, arguments); @@ -1265,7 +1295,11 @@ openerp.web.form.FieldBoolean = openerp.web.form.Field.extend({ }, set_value: function(value) { this._super.apply(this, arguments); - this.$element.find('input')[0].checked = value; + if (this.view.readonly) { + this.$element.find('span').text(value ? '✔' : '✘'); + } else { + this.$element.find('input')[0].checked = value; + } }, set_value_from_ui: function() { this.value = this.$element.find('input').is(':checked'); @@ -1347,11 +1381,17 @@ openerp.web.form.FieldSelection = openerp.web.form.Field.extend({ value = value === null ? false : value; value = value instanceof Array ? value[0] : value; this._super(value); - var index = 0; - for (var i = 0, ii = this.values.length; i < ii; i++) { - if (this.values[i][0] === value) index = i; + if (this.view.readonly) { + var option = _(this.values) + .detect(function (record) { return record[0] === value; }); + this.$element.find('div').text(option ? option[1] : this.values[0][1]); + } else { + var index = 0; + for (var i = 0, ii = this.values.length; i < ii; i++) { + if (this.values[i][0] === value) index = i; + } + this.$element.find('select')[0].selectedIndex = index; } - this.$element.find('select')[0].selectedIndex = index; }, set_value_from_ui: function() { this.value = this.values[this.$element.find('select')[0].selectedIndex][0]; @@ -1362,6 +1402,7 @@ openerp.web.form.FieldSelection = openerp.web.form.Field.extend({ this.$element.find('select').attr('disabled', this.readonly); }, validate: function() { + if (this.view.readonly) { return; } var value = this.values[this.$element.find('select')[0].selectedIndex]; this.invalid = !(value && !(this.required && value[0] === false)); }, @@ -1428,6 +1469,7 @@ openerp.web.form.FieldMany2One = openerp.web.form.Field.extend({ }, start: function() { this._super(); + if (this.view.readonly) { return; } var self = this; this.$input = this.$element.find("input"); this.$drop_down = this.$element.find(".oe-m2o-drop-down-button"); @@ -1647,6 +1689,10 @@ openerp.web.form.FieldMany2One = openerp.web.form.Field.extend({ self.update_dom(); self.on_value_changed(); var real_set_value = function(rval) { + if (self.view.readonly) { + self.$element.find('div').text(rval ? rval[1] : ''); + return; + } self.tmp_value = undefined; self.value = rval; self.original_value = undefined; diff --git a/addons/web/static/src/xml/base.xml b/addons/web/static/src/xml/base.xml index c88b0116a27..45812e9fd08 100644 --- a/addons/web/static/src/xml/base.xml +++ b/addons/web/static/src/xml/base.xml @@ -730,15 +730,23 @@
- - - + + +

+ + + + + + + + @@ -764,7 +772,15 @@

+
+ +
- + + # + @@ -790,9 +807,10 @@ -
- + + # + @@ -801,7 +819,14 @@
+