[IMP] registry: adapt LRU sizes for registries and ormcache

The registry size is now assumed to be around 10Mb, and ormcache size is
proportional to the maximum number of registries.  Statistics about ormcache
usage is shown to the log when receiving signal SIGUSR1.

openerp/modules/registry.py

@@ -305,8 +305,9 @@ class RegistryManager(object):
                     # cannot specify the memory limit soft on windows...
                     size = 42
                 else:
-                    # On average, a clean registry take 25MB of memory + cache
-                    avgsz = 30 * 1024 * 1024
+                    # A registry takes 10MB of memory on average, so we reserve
+                    # 10Mb (registry) + 5Mb (working memory) per registry
+                    avgsz = 15 * 1024 * 1024
                     size = int(config['limit_memory_soft'] / avgsz)
 
             cls._registries = LRU(size)
@@ -319,8 +320,8 @@ class RegistryManager(object):
         """
         with cls.lock():
             if cls._cache is None:
-                # we allocate one cache entry per 32KB of memory
-                size = max(8192, int(config['limit_memory_soft'] / 32768))
+                # we allocate 8192 cache entries per registry
+                size = 8192 * cls.registries.count
                 cls._cache = LRU(size)
             return cls._cache
 

openerp/service/server.py

@@ -38,7 +38,7 @@ import openerp
 from openerp.modules.registry import RegistryManager
 from openerp.release import nt_service_name
 import openerp.tools.config as config
-from openerp.tools.misc import stripped_sys_argv, dumpstacks
+from openerp.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats
 
 _logger = logging.getLogger(__name__)
 
@@ -296,6 +296,7 @@ class ThreadedServer(CommonServer):
             signal.signal(signal.SIGCHLD, self.signal_handler)
             signal.signal(signal.SIGHUP, self.signal_handler)
             signal.signal(signal.SIGQUIT, dumpstacks)
+            signal.signal(signal.SIGUSR1, log_ormcache_stats)
         elif os.name == 'nt':
             import win32api
             win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1)
@@ -389,6 +390,7 @@ class GeventServer(CommonServer):
 
         if os.name == 'posix':
             signal.signal(signal.SIGQUIT, dumpstacks)
+            signal.signal(signal.SIGUSR1, log_ormcache_stats)
 
         gevent.spawn(self.watch_parent)
         self.httpd = WSGIServer((self.interface, self.port), self.app)
@@ -510,6 +512,9 @@ class PreforkServer(CommonServer):
             elif sig == signal.SIGQUIT:
                 # dump stacks on kill -3
                 self.dumpstacks()
+            elif sig == signal.SIGUSR1:
+                # log ormcache stats on kill -SIGUSR1
+                log_ormcache_stats()
             elif sig == signal.SIGTTIN:
                 # increase number of workers
                 self.population += 1
@@ -586,6 +591,7 @@ class PreforkServer(CommonServer):
         signal.signal(signal.SIGTTIN, self.signal_handler)
         signal.signal(signal.SIGTTOU, self.signal_handler)
         signal.signal(signal.SIGQUIT, dumpstacks)
+        signal.signal(signal.SIGUSR1, log_ormcache_stats)
 
         # listen to socket
         self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

openerp/tools/cache.py

@@ -27,7 +27,7 @@ from inspect import getargspec
 import lru
 import logging
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 class ormcache(object):
@@ -47,9 +47,9 @@ class ormcache(object):
         return lookup
 
     def stat(self):
-        return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % \
+        return "lookup-stats %6d hit %6d miss %6d err %4.1f ratio" % \
             (self.stat_hit, self.stat_miss, self.stat_err,
-                (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit))
+                (100*float(self.stat_hit))/(self.stat_miss+self.stat_hit or 1))
 
     def lru(self, model):
         return model.pool.cache, (model.pool.db_name, model._name, self.method)
@@ -73,9 +73,9 @@ class ormcache(object):
         """ Remove *args entry from the cache or all keys if *args is undefined """
         d, key0 = self.lru(model)
         if args:
-            logger.warn("ormcache.clear arguments are deprecated and ignored "
-                        "(while clearing caches on (%s).%s)",
-                        model._name, self.method.__name__)
+            _logger.warn("ormcache.clear arguments are deprecated and ignored "
+                         "(while clearing caches on (%s).%s)",
+                         model._name, self.method.__name__)
         d.clear_prefix(key0)
         model.pool._any_cache_cleared = True
 
@@ -168,6 +168,25 @@ class dummy_cache(object):
         pass
 
 
+def log_ormcache_stats(sig=None, frame=None):
+    """ Log statistics of ormcache usage by database, model, and method. """
+    from openerp.modules.registry import RegistryManager
+    from collections import defaultdict
+    import threading
+
+    me = threading.currentThread()
+    entries = defaultdict(int)
+    for key in RegistryManager.cache.iterkeys():
+        entries[key[:3]] += 1
+    for (dbname, model_name, method), count in sorted(entries.items()):
+        me.dbname = dbname
+        model = RegistryManager.get(dbname)[model_name]
+        func = getattr(model, method.__name__).im_func
+        ormcache = func.clear_cache.im_self
+        _logger.info("%6d entries, %s, for %s.%s",
+            count, ormcache.stat(), model_name, method.__name__)
+
+
 # For backward compatibility
 cache = ormcache