diff --git a/openerp/modules/registry.py b/openerp/modules/registry.py index 5c9870a488a..352c6f156af 100644 --- a/openerp/modules/registry.py +++ b/openerp/modules/registry.py @@ -305,8 +305,9 @@ class RegistryManager(object): # cannot specify the memory limit soft on windows... size = 42 else: - # On average, a clean registry take 25MB of memory + cache - avgsz = 30 * 1024 * 1024 + # A registry takes 10MB of memory on average, so we reserve + # 10Mb (registry) + 5Mb (working memory) per registry + avgsz = 15 * 1024 * 1024 size = int(config['limit_memory_soft'] / avgsz) cls._registries = LRU(size) @@ -319,8 +320,8 @@ class RegistryManager(object): """ with cls.lock(): if cls._cache is None: - # we allocate one cache entry per 32KB of memory - size = max(8192, int(config['limit_memory_soft'] / 32768)) + # we allocate 8192 cache entries per registry + size = 8192 * cls.registries.count cls._cache = LRU(size) return cls._cache diff --git a/openerp/service/server.py b/openerp/service/server.py index 0ec039e83ad..b6c57fa7325 100644 --- a/openerp/service/server.py +++ b/openerp/service/server.py @@ -38,7 +38,7 @@ import openerp from openerp.modules.registry import RegistryManager from openerp.release import nt_service_name import openerp.tools.config as config -from openerp.tools.misc import stripped_sys_argv, dumpstacks +from openerp.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats _logger = logging.getLogger(__name__) @@ -296,6 +296,7 @@ class ThreadedServer(CommonServer): signal.signal(signal.SIGCHLD, self.signal_handler) signal.signal(signal.SIGHUP, self.signal_handler) signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) elif os.name == 'nt': import win32api win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1) @@ -389,6 +390,7 @@ class GeventServer(CommonServer): if os.name == 'posix': signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) gevent.spawn(self.watch_parent) self.httpd = WSGIServer((self.interface, self.port), self.app) @@ -510,6 +512,9 @@ class PreforkServer(CommonServer): elif sig == signal.SIGQUIT: # dump stacks on kill -3 self.dumpstacks() + elif sig == signal.SIGUSR1: + # log ormcache stats on kill -SIGUSR1 + log_ormcache_stats() elif sig == signal.SIGTTIN: # increase number of workers self.population += 1 @@ -586,6 +591,7 @@ class PreforkServer(CommonServer): signal.signal(signal.SIGTTIN, self.signal_handler) signal.signal(signal.SIGTTOU, self.signal_handler) signal.signal(signal.SIGQUIT, dumpstacks) + signal.signal(signal.SIGUSR1, log_ormcache_stats) # listen to socket self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/openerp/tools/cache.py b/openerp/tools/cache.py index ab387de361a..99364f8da98 100644 --- a/openerp/tools/cache.py +++ b/openerp/tools/cache.py @@ -27,7 +27,7 @@ from inspect import getargspec import lru import logging -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) class ormcache(object): @@ -47,9 +47,9 @@ class ormcache(object): return lookup def stat(self): - return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % \ + return "lookup-stats %6d hit %6d miss %6d err %4.1f ratio" % \ (self.stat_hit, self.stat_miss, self.stat_err, - (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit)) + (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit or 1)) def lru(self, model): return model.pool.cache, (model.pool.db_name, model._name, self.method) @@ -73,9 +73,9 @@ class ormcache(object): """ Remove *args entry from the cache or all keys if *args is undefined """ d, key0 = self.lru(model) if args: - logger.warn("ormcache.clear arguments are deprecated and ignored " - "(while clearing caches on (%s).%s)", - model._name, self.method.__name__) + _logger.warn("ormcache.clear arguments are deprecated and ignored " + "(while clearing caches on (%s).%s)", + model._name, self.method.__name__) d.clear_prefix(key0) model.pool._any_cache_cleared = True @@ -168,6 +168,25 @@ class dummy_cache(object): pass +def log_ormcache_stats(sig=None, frame=None): + """ Log statistics of ormcache usage by database, model, and method. """ + from openerp.modules.registry import RegistryManager + from collections import defaultdict + import threading + + me = threading.currentThread() + entries = defaultdict(int) + for key in RegistryManager.cache.iterkeys(): + entries[key[:3]] += 1 + for (dbname, model_name, method), count in sorted(entries.items()): + me.dbname = dbname + model = RegistryManager.get(dbname)[model_name] + func = getattr(model, method.__name__).im_func + ormcache = func.clear_cache.im_self + _logger.info("%6d entries, %s, for %s.%s", + count, ormcache.stat(), model_name, method.__name__) + + # For backward compatibility cache = ormcache