From e2ea691cef35941e9c0a3e807d1c303ba030663d Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 3 Mar 2015 11:26:10 +0100 Subject: [PATCH] [IMP] ormcache: turn it into a global LRU cache shared among registries The ormcache is now shared among registries. The cached methods use keys like (DBNAME, MODELNAME, METHOD, args...). This allows registries with high load to use more cache than other registries. --- openerp/addons/base/tests/test_api.py | 4 +--- openerp/models.py | 5 +---- openerp/modules/registry.py | 17 +++++++++++++++++ openerp/tools/cache.py | 23 +++++++++-------------- openerp/tools/lru.py | 8 ++++++++ 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/openerp/addons/base/tests/test_api.py b/openerp/addons/base/tests/test_api.py index 1f5203260e6..7deeafa946a 100644 --- a/openerp/addons/base/tests/test_api.py +++ b/openerp/addons/base/tests/test_api.py @@ -31,12 +31,10 @@ class TestAPI(common.TransactionCase): self.assertTrue(ids) self.assertTrue(partners) - # partners and its contents are instance of the model, and share its ormcache + # partners and its contents are instance of the model self.assertIsRecordset(partners, 'res.partner') - self.assertIs(partners._ormcache, self.env['res.partner']._ormcache) for p in partners: self.assertIsRecord(p, 'res.partner') - self.assertIs(p._ormcache, self.env['res.partner']._ormcache) self.assertEqual([p.id for p in partners], ids) self.assertEqual(self.env['res.partner'].browse(ids), partners) diff --git a/openerp/models.py b/openerp/models.py index 73782dea625..2f820cec4e2 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -827,9 +827,6 @@ class BaseModel(object): "TransientModels must have log_access turned on, " \ "in order to implement their access rights policy" - # prepare ormcache, which must be shared by all instances of the model - cls._ormcache = {} - @api.model @ormcache() def _is_an_ordinary_table(self): @@ -1827,7 +1824,7 @@ class BaseModel(object): ``tools.ormcache`` or ``tools.ormcache_multi``. """ try: - self._ormcache.clear() + self.pool.cache.clear_prefix((self.pool.db_name, self._name)) self.pool._any_cache_cleared = True except AttributeError: pass diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index c6e3e2ed1d2..5c9870a488a 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -104,6 +104,10 @@ class Registry(Mapping): """ Same as ``self[model_name]``. """ return self.models[model_name] + @lazy_property + def cache(self): + return RegistryManager.cache + @lazy_property def pure_function_fields(self): """ Return the list of pure function fields (field objects) """ @@ -287,6 +291,7 @@ class RegistryManager(object): """ _registries = None + _cache = None _lock = threading.RLock() _saved_lock = None @@ -307,6 +312,18 @@ class RegistryManager(object): cls._registries = LRU(size) return cls._registries + @classproperty + def cache(cls): + """ Return the global LRU ormcache. Its keys are tuples with the + following structure: (db_name, model_name, method, args...). + """ + with cls.lock(): + if cls._cache is None: + # we allocate one cache entry per 32KB of memory + size = max(8192, int(config['limit_memory_soft'] / 32768)) + cls._cache = LRU(size) + return cls._cache + @classmethod def lock(cls): """ Return the current registry lock. """ diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index 57dea0888c1..ab387de361a 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -52,16 +52,11 @@ class ormcache(object): (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit)) def lru(self, model): - ormcache = model._ormcache - try: - d = ormcache[self.method] - except KeyError: - d = ormcache[self.method] = lru.LRU(self.size) - return d + return model.pool.cache, (model.pool.db_name, model._name, self.method) def lookup(self, method, *args, **kwargs): - d = self.lru(args[0]) - key = args[self.skiparg:] + d, key0 = self.lru(args[0]) + key = key0 + args[self.skiparg:] try: r = d[key] self.stat_hit += 1 @@ -76,12 +71,12 @@ class ormcache(object): def clear(self, model, *args): """ Remove *args entry from the cache or all keys if *args is undefined """ - d = self.lru(model) + d, key0 = self.lru(model) if args: logger.warn("ormcache.clear arguments are deprecated and ignored " "(while clearing caches on (%s).%s)", model._name, self.method.__name__) - d.clear() + d.clear_prefix(key0) model.pool._any_cache_cleared = True @@ -97,7 +92,7 @@ class ormcache_context(ormcache): return super(ormcache_context, self).__call__(method) def lookup(self, method, *args, **kwargs): - d = self.lru(args[0]) + d, key0 = self.lru(args[0]) # Note. The decorator() wrapper (used in __call__ above) will resolve # arguments, and pass them positionally to lookup(). This is why context @@ -109,7 +104,7 @@ class ormcache_context(ormcache): ckey = [(k, context[k]) for k in self.accepted_keys if k in context] # Beware: do not take the context from args! - key = args[self.skiparg:self.context_pos] + tuple(ckey) + key = key0 + args[self.skiparg:self.context_pos] + tuple(ckey) try: r = d[key] self.stat_hit += 1 @@ -130,8 +125,8 @@ class ormcache_multi(ormcache): self.multi = multi def lookup(self, method, *args, **kwargs): - d = self.lru(args[0]) - base_key = args[self.skiparg:self.multi] + args[self.multi+1:] + d, key0 = self.lru(args[0]) + base_key = key0 + args[self.skiparg:self.multi] + args[self.multi+1:] ids = args[self.multi] result = {} missed = [] diff --git a/openerp/tools/lru.py b/openerp/tools/lru.py index 13b76f387b7..c9de62130ce 100644 --- a/openerp/tools/lru.py +++ b/openerp/tools/lru.py @@ -119,4 +119,12 @@ class LRU(object): self.first = None self.last = None + @synchronized() + def clear_prefix(self, prefix): + """ Remove from `self` all the items with the given `prefix`. """ + n = len(prefix) + for key in self.keys(): + if key[:n] == prefix: + del self[key] + # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: