odoo
/
odoo
2
0
Fork 0
Code Pull Requests Releases Activity

[IMP] Reduce considerably the loading time of a new registry. 

Loading time was mesured on the loading of a second database (identical to the
first one), i.e. by passing -d xx,yy on the command line, using cProfile. The
databases were installed with sale, mrp, and the crm.

The cProfile code is commited as part of this patch and should be removed (or
maybe guarded by some command-line flag) (just as the commenting-out of the
cron startup).

The patch was also applied on top of the trunk-simple-table-stats-vmt branch
which provides SQL queries counters.

Results indicate that the number of SQL queries are reduced from about 2100 to
27. Loading time is reduced from 1.3s to 0.26s (i.e. improved by 5).

Changes:

All calls to ir_model_fields to fetch manual (custom) fields are done in a
single call (prior to instanciate all models).

Checks for empty module descriptions are not done unless we are in init or
update mode. (The behavior was the opposite, which was probably a mistake).

Some calls to ir_translation, passing en_US because there was no lang in the
context, are not done anymore.

The improved time is also a result of a change in the decimal_precision module
where precision_get fetches all digits/applications instead of one at a time
(and thus implements its own caching instead of relying on
openerp.tools.ormache).

bzr revid: vmt@openerp.com-20121211105954-lwgs5js7yw3tzghs
This commit is contained in:
Vo Minh Thu 2012-12-11 11:59:54 +01:00
parent 88eb634fd5
commit f668a123d9
3 changed files with 39 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
openerp/cli/server.py
View File

@ -94,10 +94,11 @@ def setup_pid_file():
def preload_registry(dbname):
""" Preload a registry, and start the cron."""
try:
db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=openerp.tools.config['init'] or openerp.tools.config['update'], pooljobs=False)
update_module = True if openerp.tools.config['init'] or openerp.tools.config['update'] else False
db, registry = openerp.pooler.get_db_and_pool(dbname, update_module=update_module, pooljobs=False)
# jobs will start to be processed later, when openerp.cron.start_master_thread() is called by openerp.service.start_services()
registry.schedule_cron_jobs()
#registry.schedule_cron_jobs()
except Exception:
_logger.exception('Failed to initialize database `%s`.', dbname)
@ -258,9 +259,12 @@ def main(args):
else:
openerp.service.start_services()
import cProfile
if config['db_name']:
for dbname in config['db_name'].split(','):
preload_registry(dbname)
prof = cProfile.Profile()
prof.runcall(preload_registry, dbname)
prof.dump_stats('preload_registry_%s.profile' % dbname)
if config["stop_after_init"]:
sys.exit(0)

19
openerp/modules/loading.py
View File

@ -146,6 +146,11 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
cr.execute("select (now() at time zone 'UTC')::timestamp")
dt_before_load = cr.fetchone()[0]
pool.fields_by_model = {'whatever': {}}
cr.execute('SELECT * FROM ir_model_fields WHERE state=%s', ('manual',))
for field in cr.dictfetchall():
pool.fields_by_model.setdefault(field['model'], []).append(field)
# register, instantiate and initialize models for each modules
for index, package in enumerate(graph):
module_name = package.name
@ -159,6 +164,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
load_openerp_module(package.name)
models = pool.load(cr, package)
loaded_modules.append(package.name)
if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'):
init_module_models(cr, package.name, models)
@ -220,6 +226,8 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=
delattr(package, kind)
cr.commit()
pool.fields_by_model = {}
cr.commit()
@ -239,7 +247,7 @@ def _check_module_names(cr, module_names):
incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()])
_logger.warning('invalid module names, ignored: %s', ", ".join(incorrect_names))
def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules):
def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_modules, perform_checks):
"""Loads modules marked with ``states``, adding them to ``graph`` and
``loaded_modules`` and returns a list of installed/upgraded modules."""
processed_modules = []
@ -248,7 +256,7 @@ def load_marked_modules(cr, graph, states, force, progressdict, report, loaded_m
module_list = [name for (name,) in cr.fetchall() if name not in graph]
graph.add_modules(cr, module_list, force)
_logger.debug('Updating graph with %d more modules', len(module_list))
loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules)
loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks)
processed_modules.extend(processed)
loaded_modules.extend(loaded)
if not processed: break
@ -293,7 +301,8 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
# processed_modules: for cleanup step after install
# loaded_modules: to avoid double loading
report = pool._assertion_report
loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)
print update_module
loaded_modules, processed_modules = load_module_graph(cr, graph, status, perform_checks=update_module, report=report)
if tools.config['load_language']:
for lang in tools.config['load_language'].split(','):
@ -333,11 +342,11 @@ def load_modules(db, force_demo=False, status=None, update_module=False):
# be dropped in STEP 6 later, before restarting the loading
# process.
states_to_load = ['installed', 'to upgrade', 'to remove']
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
processed_modules.extend(processed)
if update_module:
states_to_load = ['to install']
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules)
processed = load_marked_modules(cr, graph, states_to_load, force, status, report, loaded_modules, update_module)
processed_modules.extend(processed)
# load custom models

31
openerp/osv/orm.py
View File

@ -1018,10 +1018,14 @@ class BaseModel(object):
# Load manual fields
cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields'))
if cr.fetchone():
cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual'))
for field in cr.dictfetchall():
#cr.execute("SELECT id FROM ir_model_fields WHERE name=%s AND model=%s", ('state', 'ir.model.fields'))
if True: #cr.fetchone():
if self.pool.fields_by_model:
fields__ = self.pool.fields_by_model.get(self._name, [])
else:
cr.execute('SELECT * FROM ir_model_fields WHERE model=%s AND state=%s', (self._name, 'manual'))
fields__ = cr.dictfetchall()
for field in fields__:
if field['name'] in self._columns:
continue
attrs = {
@ -3628,15 +3632,16 @@ class BaseModel(object):
else:
res = map(lambda x: {'id': x}, ids)
for f in fields_pre:
if f == self.CONCURRENCY_CHECK_FIELD:
continue
if self._columns[f].translate:
ids = [x['id'] for x in res]
#TODO: optimize out of this loop
res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context.get('lang', False) or 'en_US', ids)
for r in res:
r[f] = res_trans.get(r['id'], False) or r[f]
if context.get('lang'):
for f in fields_pre:
if f == self.CONCURRENCY_CHECK_FIELD:
continue
if self._columns[f].translate:
ids = [x['id'] for x in res]
#TODO: optimize out of this loop
res_trans = self.pool.get('ir.translation')._get_ids(cr, user, self._name+','+f, 'model', context['lang'], ids)
for r in res:
r[f] = res_trans.get(r['id'], False) or r[f]
for table in self._inherits:
col = self._inherits[table]