[IMP] Reduce considerably the loading time of a new registry.
Loading time was mesured on the loading of a second database (identical to the first one), i.e. by passing -d xx,yy on the command line, using cProfile. The databases were installed with sale, mrp, and the crm. The cProfile code is commited as part of this patch and should be removed (or maybe guarded by some command-line flag) (just as the commenting-out of the cron startup). The patch was also applied on top of the trunk-simple-table-stats-vmt branch which provides SQL queries counters. Results indicate that the number of SQL queries are reduced from about 2100 to 27. Loading time is reduced from 1.3s to 0.26s (i.e. improved by 5). Changes: All calls to ir_model_fields to fetch manual (custom) fields are done in a single call (prior to instanciate all models). Checks for empty module descriptions are not done unless we are in init or update mode. (The behavior was the opposite, which was probably a mistake). Some calls to ir_translation, passing en_US because there was no lang in the context, are not done anymore. The improved time is also a result of a change in the decimal_precision module where precision_get fetches all digits/applications instead of one at a time (and thus implements its own caching instead of relying on openerp.tools.ormache). bzr revid: vmt@openerp.com-20121211105954-lwgs5js7yw3tzghs
This commit is contained in:
parent
88eb634fd5
commit
f668a123d9
|
@ -94,10 +94,11 @@ def setup_pid_file():
|
|||
def preload_registry(dbname):
|
||||
""" Preload a registry, and start the cron."""
|
||||
try:
|
||||
db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=openerp.tools.config['init'] or openerp.tools.config['update'], pooljobs=False)
|
||||
update_module = True if openerp.tools.config['init'] or openerp.tools.config['update'] else False
|
||||
db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=update_module, pooljobs=False)
|
||||
|
||||
# jobs will start to be processed later, when openerp.cron.start_master_thread() is called by openerp.service.start_services()
|
||||
registry.schedule_cron_jobs()
|
||||
#registry.schedule_cron_jobs()
|
||||
except Exception:
|
||||
_logger.exception('Failed to initialize database `%s`.', dbname)
|
||||
|
||||
|
@ -258,9 +259,12 @@ def main(args):
|
|||
else:
|
||||
openerp.service.start_services()
|
||||
|
||||
import cProfile
|
||||
if config['db_name']:
|
||||
for dbname in config['db_name'].split(','):
|
||||
preload_registry(dbname)
|
||||
prof = cProfile.Profile()
|
||||
prof.runcall(preload_registry, dbname)
|
||||
prof.dump_stats('preload_registry_%s.profile' % dbname)
|
||||
|
||||
if config["stop_after_init"]:
|
||||
sys.exit(0)
|
||||
|
|
|
@ -146,6 +146,11 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
|
|||
cr.execute("select (now() at time zone 'UTC')::timestamp")
|
||||
dt_before_load = cr.fetchone()[0]
|
||||
|
||||
pool.fields_by_model = {'whatever': {}}
|
||||
cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
|
||||
for field in cr.dictfetchall():
|
||||
pool.fields_by_model.setdefault(field['model'], []).append(field)
|
||||
|
||||
# register, instantiate and initialize models for each modules
|
||||
for index, package in enumerate(graph):
|
||||
module_name = package.name
|
||||
|
@ -159,6 +164,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
|
|||
load_openerp_module(package.name)
|
||||
|
||||
models = pool.load(cr, package)
|
||||
|
||||
loaded_modules.append(package.name)
|
||||
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
|
||||
init_module_models(cr, package.name, models)
|
||||
|
@ -220,6 +226,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
|
|||
delattr(package, kind)
|
||||
|
||||
cr.commit()
|
||||
|
||||
pool.fields_by_model = {}
|
||||
|
||||
cr.commit()
|
||||
|
||||
|
@ -239,7 +247,7 @@ def _check_module_names(cr, module_names):
|
|||
incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
|
||||
_logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
|
||||
|
||||
def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
|
||||
def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks):
|
||||
"""Loads modules marked with ``states``, adding them to ``graph`` and
|
||||
``loaded_modules`` and returns a list of installed/upgraded modules."""
|
||||
processed_modules = []
|
||||
|
@ -248,7 +256,7 @@ def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_m
|
|||
module_list = [name for (name,) in cr.fetchall() if name not in graph]
|
||||
graph.add_modules(cr, module_list, force)
|
||||
_logger.debug('Updating graph with %d more modules', len(module_list))
|
||||
loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
|
||||
loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks)
|
||||
processed_modules.extend(processed)
|
||||
loaded_modules.extend(loaded)
|
||||
if not processed: break
|
||||
|
@ -293,7 +301,8 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
|
|||
# processed_modules: for cleanup step after install
|
||||
# loaded_modules: to avoid double loading
|
||||
report = pool._assertion_report
|
||||
loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
|
||||
print update_module
|
||||
loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
|
||||
|
||||
if tools.config['load_language']:
|
||||
for lang in tools.config['load_language'].split(','):
|
||||
|
@ -333,11 +342,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
|
|||
# be dropped in STEP 6 later, before restarting the loading
|
||||
# process.
|
||||
states_to_load = ['installed', 'to upgrade', 'to remove']
|
||||
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
|
||||
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
|
||||
processed_modules.extend(processed)
|
||||
if update_module:
|
||||
states_to_load = ['to install']
|
||||
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
|
||||
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
|
||||
processed_modules.extend(processed)
|
||||
|
||||
# load custom models
|
||||
|
|
|
@ -1018,10 +1018,14 @@ class BaseModel(object):
|
|||
|
||||
# Load manual fields
|
||||
|
||||
cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields'))
|
||||
if cr.fetchone():
|
||||
cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual'))
|
||||
for field in cr.dictfetchall():
|
||||
#cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields'))
|
||||
if True: #cr.fetchone():
|
||||
if self.pool.fields_by_model:
|
||||
fields__ = self.pool.fields_by_model.get(self._name, [])
|
||||
else:
|
||||
cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual'))
|
||||
fields__ = cr.dictfetchall()
|
||||
for field in fields__:
|
||||
if field['name'] in self._columns:
|
||||
continue
|
||||
attrs = {
|
||||
|
@ -3628,15 +3632,16 @@ class BaseModel(object):
|
|||
else:
|
||||
res = map(lambda x: {'id': x}, ids)
|
||||
|
||||
for f in fields_pre:
|
||||
if f == self.CONCURRENCY_CHECK_FIELD:
|
||||
continue
|
||||
if self._columns[f].translate:
|
||||
ids = [x['id'] for x in res]
|
||||
#TODO: optimize out of this loop
|
||||
res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context.get('lang', False) or 'en_US', ids)
|
||||
for r in res:
|
||||
r[f] = res_trans.get(r['id'], False) or r[f]
|
||||
if context.get('lang'):
|
||||
for f in fields_pre:
|
||||
if f == self.CONCURRENCY_CHECK_FIELD:
|
||||
continue
|
||||
if self._columns[f].translate:
|
||||
ids = [x['id'] for x in res]
|
||||
#TODO: optimize out of this loop
|
||||
res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context['lang'], ids)
|
||||
for r in res:
|
||||
r[f] = res_trans.get(r['id'], False) or r[f]
|
||||
|
||||
for table in self._inherits:
|
||||
col = self._inherits[table]
|
||||
|
|
Loading…
Reference in New Issue