[MERGE] added resources (virt. memory and CPU time) limits when using gunicorn.

bzr revid: vmt@openerp.com-20120120172018-dlde0c1qn6oh9922

gunicorn.conf.py

@@ -22,11 +22,15 @@ workers = 4
 # Some application-wide initialization is needed.
 on_starting = openerp.wsgi.on_starting
 when_ready = openerp.wsgi.when_ready
+pre_request = openerp.wsgi.pre_request
+post_request = openerp.wsgi.post_request
 
 # openerp request-response cycle can be quite long for
 # big reports for example
 timeout = 240
 
+max_requests = 2000
+
 # Equivalent of --load command-line option
 openerp.conf.server_wide_modules = ['web']
 
@@ -52,5 +56,4 @@ conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons
 # If --static-http-enable is used, path for the static web directory
 #conf['static_http_document_root'] = '/var/www'
 
-
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tests/addons/test_limits/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+import models
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tests/addons/test_limits/__openerp__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+{
+    'name': 'test-limits',
+    'version': '0.1',
+    'category': 'Tests',
+    'description': """A module with dummy methods.""",
+    'author': 'OpenERP SA',
+    'maintainer': 'OpenERP SA',
+    'website': 'http://www.openerp.com',
+    'depends': ['base'],
+    'data': [],
+    'installable': True,
+    'active': False,
+}
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tests/addons/test_limits/models.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+import time
+
+import openerp
+
+class m(openerp.osv.osv.Model):
+    """ This model exposes a few methods that will consume between 'almost no
+        resource' and 'a lot of resource'.
+    """
+    _name = 'test.limits.model'
+
+    def consume_nothing(self, cr, uid, context=None):
+        return True
+
+    def consume_memory(self, cr, uid, size, context=None):
+        l = [0] * size
+        return True
+
+    def leak_memory(self, cr, uid, size, context=None):
+        if not hasattr(self, 'l'):
+            self.l = []
+        self.l.append([0] * size)
+        return True
+
+    def consume_time(self, cr, uid, seconds, context=None):
+        time.sleep(seconds)
+        return True
+
+    def consume_cpu_time(self, cr, uid, seconds, context=None):
+        import os
+        t0 = time.clock()
+        t1 = time.clock()
+        while t1 - t0 < seconds:
+            for i in xrange(10000000):
+                x = i * i
+            t1 = time.clock()
+        return True
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tools/config.py

@@ -266,6 +266,20 @@ class configmanager(object):
         group.add_option("--max-cron-threads", dest="max_cron_threads", my_default=4,
                          help="Maximum number of threads processing concurrently cron jobs.",
                          type="int")
+        # TODO sensible default for the three following limits.
+        group.add_option("--virtual-memory-limit", dest="virtual_memory_limit", my_default=768 * 1024 * 1024,
+                         help="Maximum allowed virtual memory per Gunicorn process. "
+                         "When the limit is reached, any memory allocation will fail.",
+                         type="int")
+        group.add_option("--virtual-memory-reset", dest="virtual_memory_reset", my_default=640 * 1024 * 1024,
+                         help="Maximum allowed virtual memory per Gunicorn process. "
+                         "When the limit is reached, the worker will be reset after "
+                         "the current request.",
+                         type="int")
+        group.add_option("--cpu-time-limit", dest="cpu_time_limit", my_default=60,
+                         help="Maximum allowed CPU time per Gunicorn process. "
+                         "When the limit is reached, an exception is raised.",
+                         type="int")
         group.add_option("--unaccent", dest="unaccent", my_default=False, action="store_true",
                          help="Use the unaccent function provided by the database when available.")
 
@@ -371,7 +385,8 @@ class configmanager(object):
             'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog',
             'list_db', 'xmlrpcs',
             'test_file', 'test_disable', 'test_commit', 'test_report_directory',
-            'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent',
+            'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads',
+            'virtual_memory_limit', 'virtual_memory_reset', 'cpu_time_limit', 'unaccent',
         ]
 
         for arg in keys:

openerp/wsgi.py

@@ -480,12 +480,44 @@ def when_ready(server):
     # Hijack gunicorn's SIGWINCH handling; we can choose another one.
     signal.signal(signal.SIGWINCH, make_winch_handler(server))
 
+# Install limits on virtual memory and CPU time consumption.
+def pre_request(worker, req):
+    import os
+    import psutil
+    import resource
+    import signal
+    # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space
+    rss, vms = psutil.Process(os.getpid()).get_memory_info()
+    soft, hard = resource.getrlimit(resource.RLIMIT_AS)
+    resource.setrlimit(resource.RLIMIT_AS, (config['virtual_memory_limit'], hard))
+
+    r = resource.getrusage(resource.RUSAGE_SELF)
+    cpu_time = r.ru_utime + r.ru_stime
+    signal.signal(signal.SIGXCPU, time_expired)
+    soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['cpu_time_limit'], hard))
+
+# Reset the worker if it consumes too much memory (e.g. caused by a memory leak).
+def post_request(worker, req, environ):
+    import os
+    import psutil
+    rss, vms = psutil.Process(os.getpid()).get_memory_info()
+    if vms > config['virtual_memory_reset']:
+        logging.getLogger('wsgi.worker').info('Virtual memory consumption '
+            'too high, rebooting the worker.')
+        worker.alive = False # Commit suicide after the request.
+
 # Our signal handler will signal a SGIQUIT to all workers.
 def make_winch_handler(server):
     def handle_winch(sig, fram):
         server.kill_workers(signal.SIGQUIT) # This is gunicorn specific.
     return handle_winch
 
+# SIGXCPU (exceeded CPU time) signal handler will raise an exception.
+def time_expired(n, stack):
+    logging.getLogger('wsgi.worker').info('CPU time limit exceeded.')
+    raise Exception('CPU time limit exceeded.') # TODO one of openerp.exception
+
 # Kill gracefuly the workers (e.g. because we want to clear their cache).
 # This is done by signaling a SIGWINCH to the master process, so it can be
 # called by the workers themselves.