From f668a123d94e3e27ea7c1e7343ab8fafd5ceb69f Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Tue, 11 Dec 2012 11:59:54 +0100 Subject: [PATCH] [IMP] Reduce considerably the loading time of a new registry. Loading time was mesured on the loading of a second database (identical to the first one), i.e. by passing -d xx,yy on the command line, using cProfile. The databases were installed with sale, mrp, and the crm. The cProfile code is commited as part of this patch and should be removed (or maybe guarded by some command-line flag) (just as the commenting-out of the cron startup). The patch was also applied on top of the trunk-simple-table-stats-vmt branch which provides SQL queries counters. Results indicate that the number of SQL queries are reduced from about 2100 to 27. Loading time is reduced from 1.3s to 0.26s (i.e. improved by 5). Changes: All calls to ir_model_fields to fetch manual (custom) fields are done in a single call (prior to instanciate all models). Checks for empty module descriptions are not done unless we are in init or update mode. (The behavior was the opposite, which was probably a mistake). Some calls to ir_translation, passing en_US because there was no lang in the context, are not done anymore. The improved time is also a result of a change in the decimal_precision module where precision_get fetches all digits/applications instead of one at a time (and thus implements its own caching instead of relying on openerp.tools.ormache). bzr revid: vmt@openerp.com-20121211105954-lwgs5js7yw3tzghs --- openerp/cli/server.py | 10 +++++++--- openerp/modules/loading.py | 19 ++++++++++++++----- openerp/osv/orm.py | 31 ++++++++++++++++++------------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/openerp/cli/server.py b/openerp/cli/server.py index f030f801967..67ce518155c 100644 --- a/openerp/cli/server.py +++ b/openerp/cli/server.py @@ -94,10 +94,11 @@ def setup_pid_file(): def preload_registry(dbname): """ Preload a registry, and start the cron.""" try: - db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=openerp.tools.config['init'] or openerp.tools.config['update'], pooljobs=False) + update_module = True if openerp.tools.config['init'] or openerp.tools.config['update'] else False + db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=update_module, pooljobs=False) # jobs will start to be processed later, when openerp.cron.start_master_thread() is called by openerp.service.start_services() - registry.schedule_cron_jobs() + #registry.schedule_cron_jobs() except Exception: _logger.exception('Failed to initialize database `%s`.', dbname) @@ -258,9 +259,12 @@ def main(args): else: openerp.service.start_services() + import cProfile if config['db_name']: for dbname in config['db_name'].split(','): - preload_registry(dbname) + prof = cProfile.Profile() + prof.runcall(preload_registry, dbname) + prof.dump_stats('preload_registry_%s.profile' % dbname) if config["stop_after_init"]: sys.exit(0) diff --git a/openerp/modules/loading.py b/openerp/modules/loading.py index e7a6c50286a..48f816035ff 100644 --- a/openerp/modules/loading.py +++ b/openerp/modules/loading.py @@ -146,6 +146,11 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= cr.execute("select (now() at time zone 'UTC')::timestamp") dt_before_load = cr.fetchone()[0] + pool.fields_by_model = {'whatever': {}} + cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',)) + for field in cr.dictfetchall(): + pool.fields_by_model.setdefault(field['model'], []).append(field) + # register, instantiate and initialize models for each modules for index, package in enumerate(graph): module_name = package.name @@ -159,6 +164,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= load_openerp_module(package.name) models = pool.load(cr, package) + loaded_modules.append(package.name) if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): init_module_models(cr, package.name, models) @@ -220,6 +226,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= delattr(package, kind) cr.commit() + + pool.fields_by_model = {} cr.commit() @@ -239,7 +247,7 @@ def _check_module_names(cr, module_names): incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) _logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) -def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules): +def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks): """Loads modules marked with ``states``, adding them to ``graph`` and ``loaded_modules`` and returns a list of installed/upgraded modules.""" processed_modules = [] @@ -248,7 +256,7 @@ def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_m module_list = [name for (name,) in cr.fetchall() if name not in graph] graph.add_modules(cr, module_list, force) _logger.debug('Updating graph with %d more modules', len(module_list)) - loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules) + loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks) processed_modules.extend(processed) loaded_modules.extend(loaded) if not processed: break @@ -293,7 +301,8 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # processed_modules: for cleanup step after install # loaded_modules: to avoid double loading report = pool._assertion_report - loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report) + print update_module + loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report) if tools.config['load_language']: for lang in tools.config['load_language'].split(','): @@ -333,11 +342,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False): # be dropped in STEP 6 later, before restarting the loading # process. states_to_load = ['installed', 'to upgrade', 'to remove'] - processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules) + processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module) processed_modules.extend(processed) if update_module: states_to_load = ['to install'] - processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules) + processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module) processed_modules.extend(processed) # load custom models diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 060d6da0277..4ba3d24b4c4 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -1018,10 +1018,14 @@ class BaseModel(object): # Load manual fields - cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields')) - if cr.fetchone(): - cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) - for field in cr.dictfetchall(): + #cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields')) + if True: #cr.fetchone(): + if self.pool.fields_by_model: + fields__ = self.pool.fields_by_model.get(self._name, []) + else: + cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual')) + fields__ = cr.dictfetchall() + for field in fields__: if field['name'] in self._columns: continue attrs = { @@ -3628,15 +3632,16 @@ class BaseModel(object): else: res = map(lambda x: {'id': x}, ids) - for f in fields_pre: - if f == self.CONCURRENCY_CHECK_FIELD: - continue - if self._columns[f].translate: - ids = [x['id'] for x in res] - #TODO: optimize out of this loop - res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context.get('lang', False) or 'en_US', ids) - for r in res: - r[f] = res_trans.get(r['id'], False) or r[f] + if context.get('lang'): + for f in fields_pre: + if f == self.CONCURRENCY_CHECK_FIELD: + continue + if self._columns[f].translate: + ids = [x['id'] for x in res] + #TODO: optimize out of this loop + res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context['lang'], ids) + for r in res: + r[f] = res_trans.get(r['id'], False) or r[f] for table in self._inherits: col = self._inherits[table]