[FIX] core: ormcache is now per regisry.
A cross-registry cache was introduced by e2ea691ce
.
The initial idea was praiseworthy but sub-optimal for servers with a
lot of registries.
When there is lot of registry loaded, the cache size was huge and
clearing some entries took a lot of CPU time, increasing the chances
of timeout.
Also, the cache was not cleaned when a registry is removed from
registry LRU (this operation would also consume time).
This commit is contained in:
parent
69890cf33e
commit
14fd77c132
|
@ -1814,7 +1814,7 @@ class BaseModel(object):
|
||||||
``tools.ormcache`` or ``tools.ormcache_multi``.
|
``tools.ormcache`` or ``tools.ormcache_multi``.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self.pool.cache.clear_prefix((self.pool.db_name, self._name))
|
self.pool.cache.clear_prefix((self._name,))
|
||||||
self.pool._any_cache_cleared = True
|
self.pool._any_cache_cleared = True
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -73,6 +73,7 @@ class Registry(Mapping):
|
||||||
self.base_registry_signaling_sequence = None
|
self.base_registry_signaling_sequence = None
|
||||||
self.base_cache_signaling_sequence = None
|
self.base_cache_signaling_sequence = None
|
||||||
|
|
||||||
|
self.cache = LRU(8192)
|
||||||
# Flag indicating if at least one model cache has been cleared.
|
# Flag indicating if at least one model cache has been cleared.
|
||||||
# Useful only in a multi-process context.
|
# Useful only in a multi-process context.
|
||||||
self._any_cache_cleared = False
|
self._any_cache_cleared = False
|
||||||
|
@ -104,10 +105,6 @@ class Registry(Mapping):
|
||||||
""" Same as ``self[model_name]``. """
|
""" Same as ``self[model_name]``. """
|
||||||
return self.models[model_name]
|
return self.models[model_name]
|
||||||
|
|
||||||
@lazy_property
|
|
||||||
def cache(self):
|
|
||||||
return RegistryManager.cache
|
|
||||||
|
|
||||||
@lazy_property
|
@lazy_property
|
||||||
def pure_function_fields(self):
|
def pure_function_fields(self):
|
||||||
""" Return the list of pure function fields (field objects) """
|
""" Return the list of pure function fields (field objects) """
|
||||||
|
@ -291,7 +288,6 @@ class RegistryManager(object):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
_registries = None
|
_registries = None
|
||||||
_cache = None
|
|
||||||
_lock = threading.RLock()
|
_lock = threading.RLock()
|
||||||
_saved_lock = None
|
_saved_lock = None
|
||||||
|
|
||||||
|
@ -313,18 +309,6 @@ class RegistryManager(object):
|
||||||
cls._registries = LRU(size)
|
cls._registries = LRU(size)
|
||||||
return cls._registries
|
return cls._registries
|
||||||
|
|
||||||
@classproperty
|
|
||||||
def cache(cls):
|
|
||||||
""" Return the global LRU ormcache. Its keys are tuples with the
|
|
||||||
following structure: (db_name, model_name, method, args...).
|
|
||||||
"""
|
|
||||||
with cls.lock():
|
|
||||||
if cls._cache is None:
|
|
||||||
# we allocate 8192 cache entries per registry
|
|
||||||
size = 8192 * cls.registries.count
|
|
||||||
cls._cache = LRU(size)
|
|
||||||
return cls._cache
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def lock(cls):
|
def lock(cls):
|
||||||
""" Return the current registry lock. """
|
""" Return the current registry lock. """
|
||||||
|
|
|
@ -59,7 +59,7 @@ class ormcache(object):
|
||||||
return lookup
|
return lookup
|
||||||
|
|
||||||
def lru(self, model):
|
def lru(self, model):
|
||||||
return model.pool.cache, (model.pool.db_name, model._name, self.method)
|
return model.pool.cache, (model._name, self.method)
|
||||||
|
|
||||||
def lookup(self, method, *args, **kwargs):
|
def lookup(self, method, *args, **kwargs):
|
||||||
d, key0 = self.lru(args[0])
|
d, key0 = self.lru(args[0])
|
||||||
|
@ -181,16 +181,19 @@ def log_ormcache_stats(sig=None, frame=None):
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
me = threading.currentThread()
|
me = threading.currentThread()
|
||||||
|
me_dbname = me.dbname
|
||||||
entries = defaultdict(int)
|
entries = defaultdict(int)
|
||||||
for key in RegistryManager.cache.iterkeys():
|
for reg in RegistryManager.registries.itervalues():
|
||||||
entries[key[:3]] += 1
|
for key in reg.cache.iterkeys():
|
||||||
|
entries[key[:3]] += 1
|
||||||
for key, count in sorted(entries.items()):
|
for key, count in sorted(entries.items()):
|
||||||
dbname, model_name, method = key
|
dbname, model_name, method = key
|
||||||
me.dbname = dbname
|
me.dbname = dbname
|
||||||
stat = STAT[key]
|
stat = STAT[key]
|
||||||
_logger.info("%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s",
|
_logger.info("%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s",
|
||||||
count, stat.hit, stat.miss, stat.err, stat.ratio, model_name, method.__name__)
|
count, stat.hit, stat.miss, stat.err, stat.ratio, model_name, method.__name__)
|
||||||
|
|
||||||
|
me.dbname = me_dbname
|
||||||
|
|
||||||
# For backward compatibility
|
# For backward compatibility
|
||||||
cache = ormcache
|
cache = ormcache
|
||||||
|
|
|
@ -123,7 +123,7 @@ class LRU(object):
|
||||||
def clear_prefix(self, prefix):
|
def clear_prefix(self, prefix):
|
||||||
""" Remove from `self` all the items with the given `prefix`. """
|
""" Remove from `self` all the items with the given `prefix`. """
|
||||||
n = len(prefix)
|
n = len(prefix)
|
||||||
for key in self.keys():
|
for key in self.iterkeys():
|
||||||
if key[:n] == prefix:
|
if key[:n] == prefix:
|
||||||
del self[key]
|
del self[key]
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue