[FIX] core: ormcache is now per regisry.
A cross-registry cache was introduced by e2ea691ce
.
The initial idea was praiseworthy but sub-optimal for servers with a
lot of registries.
When there is lot of registry loaded, the cache size was huge and
clearing some entries took a lot of CPU time, increasing the chances
of timeout.
Also, the cache was not cleaned when a registry is removed from
registry LRU (this operation would also consume time).
This commit is contained in:
parent
69890cf33e
commit
14fd77c132
|
@ -1814,7 +1814,7 @@ class BaseModel(object):
|
|||
``tools.ormcache`` or ``tools.ormcache_multi``.
|
||||
"""
|
||||
try:
|
||||
self.pool.cache.clear_prefix((self.pool.db_name, self._name))
|
||||
self.pool.cache.clear_prefix((self._name,))
|
||||
self.pool._any_cache_cleared = True
|
||||
except AttributeError:
|
||||
pass
|
||||
|
|
|
@ -73,6 +73,7 @@ class Registry(Mapping):
|
|||
self.base_registry_signaling_sequence = None
|
||||
self.base_cache_signaling_sequence = None
|
||||
|
||||
self.cache = LRU(8192)
|
||||
# Flag indicating if at least one model cache has been cleared.
|
||||
# Useful only in a multi-process context.
|
||||
self._any_cache_cleared = False
|
||||
|
@ -104,10 +105,6 @@ class Registry(Mapping):
|
|||
""" Same as ``self[model_name]``. """
|
||||
return self.models[model_name]
|
||||
|
||||
@lazy_property
|
||||
def cache(self):
|
||||
return RegistryManager.cache
|
||||
|
||||
@lazy_property
|
||||
def pure_function_fields(self):
|
||||
""" Return the list of pure function fields (field objects) """
|
||||
|
@ -291,7 +288,6 @@ class RegistryManager(object):
|
|||
|
||||
"""
|
||||
_registries = None
|
||||
_cache = None
|
||||
_lock = threading.RLock()
|
||||
_saved_lock = None
|
||||
|
||||
|
@ -313,18 +309,6 @@ class RegistryManager(object):
|
|||
cls._registries = LRU(size)
|
||||
return cls._registries
|
||||
|
||||
@classproperty
|
||||
def cache(cls):
|
||||
""" Return the global LRU ormcache. Its keys are tuples with the
|
||||
following structure: (db_name, model_name, method, args...).
|
||||
"""
|
||||
with cls.lock():
|
||||
if cls._cache is None:
|
||||
# we allocate 8192 cache entries per registry
|
||||
size = 8192 * cls.registries.count
|
||||
cls._cache = LRU(size)
|
||||
return cls._cache
|
||||
|
||||
@classmethod
|
||||
def lock(cls):
|
||||
""" Return the current registry lock. """
|
||||
|
|
|
@ -59,7 +59,7 @@ class ormcache(object):
|
|||
return lookup
|
||||
|
||||
def lru(self, model):
|
||||
return model.pool.cache, (model.pool.db_name, model._name, self.method)
|
||||
return model.pool.cache, (model._name, self.method)
|
||||
|
||||
def lookup(self, method, *args, **kwargs):
|
||||
d, key0 = self.lru(args[0])
|
||||
|
@ -181,8 +181,10 @@ def log_ormcache_stats(sig=None, frame=None):
|
|||
import threading
|
||||
|
||||
me = threading.currentThread()
|
||||
me_dbname = me.dbname
|
||||
entries = defaultdict(int)
|
||||
for key in RegistryManager.cache.iterkeys():
|
||||
for reg in RegistryManager.registries.itervalues():
|
||||
for key in reg.cache.iterkeys():
|
||||
entries[key[:3]] += 1
|
||||
for key, count in sorted(entries.items()):
|
||||
dbname, model_name, method = key
|
||||
|
@ -191,6 +193,7 @@ def log_ormcache_stats(sig=None, frame=None):
|
|||
_logger.info("%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s",
|
||||
count, stat.hit, stat.miss, stat.err, stat.ratio, model_name, method.__name__)
|
||||
|
||||
me.dbname = me_dbname
|
||||
|
||||
# For backward compatibility
|
||||
cache = ormcache
|
||||
|
|
|
@ -123,7 +123,7 @@ class LRU(object):
|
|||
def clear_prefix(self, prefix):
|
||||
""" Remove from `self` all the items with the given `prefix`. """
|
||||
n = len(prefix)
|
||||
for key in self.keys():
|
||||
for key in self.iterkeys():
|
||||
if key[:n] == prefix:
|
||||
del self[key]
|
||||
|
||||
|
|
Loading…
Reference in New Issue