[IMP] ormcache: turn it into a global LRU cache shared among registries

The ormcache is now shared among registries.  The cached methods use keys like
(DBNAME, MODELNAME, METHOD, args...).  This allows registries with high load to
use more cache than other registries.

openerp/addons/base/tests/test_api.py

@@ -31,12 +31,10 @@ class TestAPI(common.TransactionCase):
         self.assertTrue(ids)
         self.assertTrue(partners)
 
-        # partners and its contents are instance of the model, and share its ormcache
+        # partners and its contents are instance of the model
         self.assertIsRecordset(partners, 'res.partner')
-        self.assertIs(partners._ormcache, self.env['res.partner']._ormcache)
         for p in partners:
             self.assertIsRecord(p, 'res.partner')
-            self.assertIs(p._ormcache, self.env['res.partner']._ormcache)
 
         self.assertEqual([p.id for p in partners], ids)
         self.assertEqual(self.env['res.partner'].browse(ids), partners)

openerp/models.py

@@ -827,9 +827,6 @@ class BaseModel(object):
                 "TransientModels must have log_access turned on, " \
                 "in order to implement their access rights policy"
 
-        # prepare ormcache, which must be shared by all instances of the model
-        cls._ormcache = {}
-
     @api.model
     @ormcache()
     def _is_an_ordinary_table(self):
@@ -1827,7 +1824,7 @@ class BaseModel(object):
         ``tools.ormcache`` or ``tools.ormcache_multi``.
         """
         try:
-            self._ormcache.clear()
+            self.pool.cache.clear_prefix((self.pool.db_name, self._name))
             self.pool._any_cache_cleared = True
         except AttributeError:
             pass

openerp/modules/registry.py

@@ -104,6 +104,10 @@ class Registry(Mapping):
         """ Same as ``self[model_name]``. """
         return self.models[model_name]
 
+    @lazy_property
+    def cache(self):
+        return RegistryManager.cache
+
     @lazy_property
     def pure_function_fields(self):
         """ Return the list of pure function fields (field objects) """
@@ -287,6 +291,7 @@ class RegistryManager(object):
 
     """
     _registries = None
+    _cache = None
     _lock = threading.RLock()
     _saved_lock = None
 
@@ -307,6 +312,18 @@ class RegistryManager(object):
             cls._registries = LRU(size)
         return cls._registries
 
+    @classproperty
+    def cache(cls):
+        """ Return the global LRU ormcache. Its keys are tuples with the
+        following structure: (db_name, model_name, method, args...).
+        """
+        with cls.lock():
+            if cls._cache is None:
+                # we allocate one cache entry per 32KB of memory
+                size = max(8192, int(config['limit_memory_soft'] / 32768))
+                cls._cache = LRU(size)
+            return cls._cache
+
     @classmethod
     def lock(cls):
         """ Return the current registry lock. """

openerp/tools/cache.py

@@ -52,16 +52,11 @@ class ormcache(object):
                 (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit))
 
     def lru(self, model):
-        ormcache = model._ormcache
-        try:
-            d = ormcache[self.method]
-        except KeyError:
-            d = ormcache[self.method] = lru.LRU(self.size)
-        return d
+        return model.pool.cache, (model.pool.db_name, model._name, self.method)
 
     def lookup(self, method, *args, **kwargs):
-        d = self.lru(args[0])
-        key = args[self.skiparg:]
+        d, key0 = self.lru(args[0])
+        key = key0 + args[self.skiparg:]
         try:
             r = d[key]
             self.stat_hit += 1
@@ -76,12 +71,12 @@ class ormcache(object):
 
     def clear(self, model, *args):
         """ Remove *args entry from the cache or all keys if *args is undefined """
-        d = self.lru(model)
+        d, key0 = self.lru(model)
         if args:
             logger.warn("ormcache.clear arguments are deprecated and ignored "
                         "(while clearing caches on (%s).%s)",
                         model._name, self.method.__name__)
-        d.clear()
+        d.clear_prefix(key0)
         model.pool._any_cache_cleared = True
 
 
@@ -97,7 +92,7 @@ class ormcache_context(ormcache):
         return super(ormcache_context, self).__call__(method)
 
     def lookup(self, method, *args, **kwargs):
-        d = self.lru(args[0])
+        d, key0 = self.lru(args[0])
 
         # Note. The decorator() wrapper (used in __call__ above) will resolve
         # arguments, and pass them positionally to lookup(). This is why context
@@ -109,7 +104,7 @@ class ormcache_context(ormcache):
         ckey = [(k, context[k]) for k in self.accepted_keys if k in context]
 
         # Beware: do not take the context from args!
-        key = args[self.skiparg:self.context_pos] + tuple(ckey)
+        key = key0 + args[self.skiparg:self.context_pos] + tuple(ckey)
         try:
             r = d[key]
             self.stat_hit += 1
@@ -130,8 +125,8 @@ class ormcache_multi(ormcache):
         self.multi = multi
 
     def lookup(self, method, *args, **kwargs):
-        d = self.lru(args[0])
-        base_key = args[self.skiparg:self.multi] + args[self.multi+1:]
+        d, key0 = self.lru(args[0])
+        base_key = key0 + args[self.skiparg:self.multi] + args[self.multi+1:]
         ids = args[self.multi]
         result = {}
         missed = []

openerp/tools/lru.py

@@ -119,4 +119,12 @@ class LRU(object):
         self.first = None
         self.last = None
 
+    @synchronized()
+    def clear_prefix(self, prefix):
+        """ Remove from `self` all the items with the given `prefix`. """
+        n = len(prefix)
+        for key in self.keys():
+            if key[:n] == prefix:
+                del self[key]
+
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: