[IMP] ormcache: turn it into a global LRU cache shared among registries
The ormcache is now shared among registries. The cached methods use keys like (DBNAME, MODELNAME, METHOD, args...). This allows registries with high load to use more cache than other registries.
This commit is contained in:
parent
e3a80afbaf
commit
e2ea691cef
|
@ -31,12 +31,10 @@ class TestAPI(common.TransactionCase):
|
||||||
self.assertTrue(ids)
|
self.assertTrue(ids)
|
||||||
self.assertTrue(partners)
|
self.assertTrue(partners)
|
||||||
|
|
||||||
# partners and its contents are instance of the model, and share its ormcache
|
# partners and its contents are instance of the model
|
||||||
self.assertIsRecordset(partners, 'res.partner')
|
self.assertIsRecordset(partners, 'res.partner')
|
||||||
self.assertIs(partners._ormcache, self.env['res.partner']._ormcache)
|
|
||||||
for p in partners:
|
for p in partners:
|
||||||
self.assertIsRecord(p, 'res.partner')
|
self.assertIsRecord(p, 'res.partner')
|
||||||
self.assertIs(p._ormcache, self.env['res.partner']._ormcache)
|
|
||||||
|
|
||||||
self.assertEqual([p.id for p in partners], ids)
|
self.assertEqual([p.id for p in partners], ids)
|
||||||
self.assertEqual(self.env['res.partner'].browse(ids), partners)
|
self.assertEqual(self.env['res.partner'].browse(ids), partners)
|
||||||
|
|
|
@ -827,9 +827,6 @@ class BaseModel(object):
|
||||||
"TransientModels must have log_access turned on, " \
|
"TransientModels must have log_access turned on, " \
|
||||||
"in order to implement their access rights policy"
|
"in order to implement their access rights policy"
|
||||||
|
|
||||||
# prepare ormcache, which must be shared by all instances of the model
|
|
||||||
cls._ormcache = {}
|
|
||||||
|
|
||||||
@api.model
|
@api.model
|
||||||
@ormcache()
|
@ormcache()
|
||||||
def _is_an_ordinary_table(self):
|
def _is_an_ordinary_table(self):
|
||||||
|
@ -1827,7 +1824,7 @@ class BaseModel(object):
|
||||||
``tools.ormcache`` or ``tools.ormcache_multi``.
|
``tools.ormcache`` or ``tools.ormcache_multi``.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self._ormcache.clear()
|
self.pool.cache.clear_prefix((self.pool.db_name, self._name))
|
||||||
self.pool._any_cache_cleared = True
|
self.pool._any_cache_cleared = True
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -104,6 +104,10 @@ class Registry(Mapping):
|
||||||
""" Same as ``self[model_name]``. """
|
""" Same as ``self[model_name]``. """
|
||||||
return self.models[model_name]
|
return self.models[model_name]
|
||||||
|
|
||||||
|
@lazy_property
|
||||||
|
def cache(self):
|
||||||
|
return RegistryManager.cache
|
||||||
|
|
||||||
@lazy_property
|
@lazy_property
|
||||||
def pure_function_fields(self):
|
def pure_function_fields(self):
|
||||||
""" Return the list of pure function fields (field objects) """
|
""" Return the list of pure function fields (field objects) """
|
||||||
|
@ -287,6 +291,7 @@ class RegistryManager(object):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
_registries = None
|
_registries = None
|
||||||
|
_cache = None
|
||||||
_lock = threading.RLock()
|
_lock = threading.RLock()
|
||||||
_saved_lock = None
|
_saved_lock = None
|
||||||
|
|
||||||
|
@ -307,6 +312,18 @@ class RegistryManager(object):
|
||||||
cls._registries = LRU(size)
|
cls._registries = LRU(size)
|
||||||
return cls._registries
|
return cls._registries
|
||||||
|
|
||||||
|
@classproperty
|
||||||
|
def cache(cls):
|
||||||
|
""" Return the global LRU ormcache. Its keys are tuples with the
|
||||||
|
following structure: (db_name, model_name, method, args...).
|
||||||
|
"""
|
||||||
|
with cls.lock():
|
||||||
|
if cls._cache is None:
|
||||||
|
# we allocate one cache entry per 32KB of memory
|
||||||
|
size = max(8192, int(config['limit_memory_soft'] / 32768))
|
||||||
|
cls._cache = LRU(size)
|
||||||
|
return cls._cache
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def lock(cls):
|
def lock(cls):
|
||||||
""" Return the current registry lock. """
|
""" Return the current registry lock. """
|
||||||
|
|
|
@ -52,16 +52,11 @@ class ormcache(object):
|
||||||
(100*float(self.stat_hit))/(self.stat_miss+self.stat_hit))
|
(100*float(self.stat_hit))/(self.stat_miss+self.stat_hit))
|
||||||
|
|
||||||
def lru(self, model):
|
def lru(self, model):
|
||||||
ormcache = model._ormcache
|
return model.pool.cache, (model.pool.db_name, model._name, self.method)
|
||||||
try:
|
|
||||||
d = ormcache[self.method]
|
|
||||||
except KeyError:
|
|
||||||
d = ormcache[self.method] = lru.LRU(self.size)
|
|
||||||
return d
|
|
||||||
|
|
||||||
def lookup(self, method, *args, **kwargs):
|
def lookup(self, method, *args, **kwargs):
|
||||||
d = self.lru(args[0])
|
d, key0 = self.lru(args[0])
|
||||||
key = args[self.skiparg:]
|
key = key0 + args[self.skiparg:]
|
||||||
try:
|
try:
|
||||||
r = d[key]
|
r = d[key]
|
||||||
self.stat_hit += 1
|
self.stat_hit += 1
|
||||||
|
@ -76,12 +71,12 @@ class ormcache(object):
|
||||||
|
|
||||||
def clear(self, model, *args):
|
def clear(self, model, *args):
|
||||||
""" Remove *args entry from the cache or all keys if *args is undefined """
|
""" Remove *args entry from the cache or all keys if *args is undefined """
|
||||||
d = self.lru(model)
|
d, key0 = self.lru(model)
|
||||||
if args:
|
if args:
|
||||||
logger.warn("ormcache.clear arguments are deprecated and ignored "
|
logger.warn("ormcache.clear arguments are deprecated and ignored "
|
||||||
"(while clearing caches on (%s).%s)",
|
"(while clearing caches on (%s).%s)",
|
||||||
model._name, self.method.__name__)
|
model._name, self.method.__name__)
|
||||||
d.clear()
|
d.clear_prefix(key0)
|
||||||
model.pool._any_cache_cleared = True
|
model.pool._any_cache_cleared = True
|
||||||
|
|
||||||
|
|
||||||
|
@ -97,7 +92,7 @@ class ormcache_context(ormcache):
|
||||||
return super(ormcache_context, self).__call__(method)
|
return super(ormcache_context, self).__call__(method)
|
||||||
|
|
||||||
def lookup(self, method, *args, **kwargs):
|
def lookup(self, method, *args, **kwargs):
|
||||||
d = self.lru(args[0])
|
d, key0 = self.lru(args[0])
|
||||||
|
|
||||||
# Note. The decorator() wrapper (used in __call__ above) will resolve
|
# Note. The decorator() wrapper (used in __call__ above) will resolve
|
||||||
# arguments, and pass them positionally to lookup(). This is why context
|
# arguments, and pass them positionally to lookup(). This is why context
|
||||||
|
@ -109,7 +104,7 @@ class ormcache_context(ormcache):
|
||||||
ckey = [(k, context[k]) for k in self.accepted_keys if k in context]
|
ckey = [(k, context[k]) for k in self.accepted_keys if k in context]
|
||||||
|
|
||||||
# Beware: do not take the context from args!
|
# Beware: do not take the context from args!
|
||||||
key = args[self.skiparg:self.context_pos] + tuple(ckey)
|
key = key0 + args[self.skiparg:self.context_pos] + tuple(ckey)
|
||||||
try:
|
try:
|
||||||
r = d[key]
|
r = d[key]
|
||||||
self.stat_hit += 1
|
self.stat_hit += 1
|
||||||
|
@ -130,8 +125,8 @@ class ormcache_multi(ormcache):
|
||||||
self.multi = multi
|
self.multi = multi
|
||||||
|
|
||||||
def lookup(self, method, *args, **kwargs):
|
def lookup(self, method, *args, **kwargs):
|
||||||
d = self.lru(args[0])
|
d, key0 = self.lru(args[0])
|
||||||
base_key = args[self.skiparg:self.multi] + args[self.multi+1:]
|
base_key = key0 + args[self.skiparg:self.multi] + args[self.multi+1:]
|
||||||
ids = args[self.multi]
|
ids = args[self.multi]
|
||||||
result = {}
|
result = {}
|
||||||
missed = []
|
missed = []
|
||||||
|
|
|
@ -119,4 +119,12 @@ class LRU(object):
|
||||||
self.first = None
|
self.first = None
|
||||||
self.last = None
|
self.last = None
|
||||||
|
|
||||||
|
@synchronized()
|
||||||
|
def clear_prefix(self, prefix):
|
||||||
|
""" Remove from `self` all the items with the given `prefix`. """
|
||||||
|
n = len(prefix)
|
||||||
|
for key in self.keys():
|
||||||
|
if key[:n] == prefix:
|
||||||
|
del self[key]
|
||||||
|
|
||||||
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
|
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
|
||||||
|
|
Loading…
Reference in New Issue