[IMP] gunicorn: add CPU and memory limits.

bzr revid: vmt@openerp.com-20120120114612-xowu57yy3f5uxi0j

gunicorn.conf.py

@@ -17,7 +17,7 @@ pidfile = '.gunicorn.pid'
 # Gunicorn recommends 2-4 x number_of_cpu_cores, but
 # you'll want to vary this a bit to find the best for your
 # particular work load.
-workers = 4
+workers = 1
 
 # Some application-wide initialization is needed.
 on_starting = openerp.wsgi.on_starting
@@ -27,6 +27,8 @@ when_ready = openerp.wsgi.when_ready
 # big reports for example
 timeout = 240
 
+#max_requests = 150
+
 # Equivalent of --load command-line option
 openerp.conf.server_wide_modules = ['web']
 
@@ -35,7 +37,7 @@ conf = openerp.tools.config
 
 # Path to the OpenERP Addons repository (comma-separated for
 # multiple locations)
-conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons'
+conf['addons_path'] = '/home/thu/repos/addons/trunk,/home/thu/repos/web/trunk/addons,/home/thu/repos/server/trunk-limits/openerp/tests/addons'
 
 # Optional database config if not using local socket
 #conf['db_name'] = 'mycompany'
@@ -52,5 +54,33 @@ conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons
 # If --static-http-enable is used, path for the static web directory
 #conf['static_http_document_root'] = '/var/www'
 
+def time_expired(n, stack):
+    import os
+    import time
+    print '>>> [%s] time ran out.' % (os.getpid())
+    raise Exception('(time ran out)')
+
+def pre_request(worker, req):
+    import os
+    import psutil
+    import resource
+    import signal
+    # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space
+    rss, vms = psutil.Process(os.getpid()).get_memory_info()
+    soft, hard = resource.getrlimit(resource.RLIMIT_AS)
+    print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment)
+    print ">>>>>>   %s" % (req.body,)
+    # Let's say 512MB is ok, 768 is a hard limit (will raise MemoryError),
+    # 640 will nicely restart the process.
+    resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, hard))
+    if vms > 640 * 1024 * 1024:
+        print ">>> Worker eating too much memory, reset it after the request."
+        worker.alive = False # Commit suicide after the request.
+
+    r = resource.getrusage(resource.RUSAGE_SELF)
+    cpu_time = r.ru_utime + r.ru_stime
+    signal.signal(signal.SIGXCPU, time_expired)
+    soft, hard = resource.getrlimit(resource.RLIMIT_CPU)
+    resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + 15, hard))
 
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tests/addons/test_limits/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+import models
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tests/addons/test_limits/__openerp__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+{
+    'name': 'test-limits',
+    'version': '0.1',
+    'category': 'Tests',
+    'description': """A module with dummy methods.""",
+    'author': 'OpenERP SA',
+    'maintainer': 'OpenERP SA',
+    'website': 'http://www.openerp.com',
+    'depends': ['base'],
+    'data': [],
+    'installable': True,
+    'active': False,
+}
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/tests/addons/test_limits/models.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+import time
+
+import openerp
+
+class m(openerp.osv.osv.Model):
+    """ This model exposes a few methods that will consume between 'almost no
+        resource' and 'a lot of resource'.
+    """
+    _name = 'test.limits.model'
+
+    def consume_nothing(self, cr, uid, context=None):
+        return True
+
+    def consume_memory(self, cr, uid, size, context=None):
+        l = [0] * size
+        return True
+
+    def leak_memory(self, cr, uid, size, context=None):
+        if not hasattr(self, 'l'):
+            self.l = []
+        self.l.append([0] * size)
+        print ">>>", len(self.l)
+        return True
+
+    def consume_time(self, cr, uid, seconds, context=None):
+        time.sleep(seconds)
+        return True
+
+    def consume_cpu_time(self, cr, uid, seconds, context=None):
+        import os
+        t0 = time.clock()
+        t1 = time.clock()
+#        try:
+        while t1 - t0 < seconds:
+            print "[%s] ..." % os.getpid()
+            for i in xrange(10000000):
+                x = i * i
+            t1 = time.clock()
+#        except Exception, e:
+#            print "+++", e
+        return True
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/wsgi.py

@@ -460,6 +460,7 @@ def on_starting(server):
     openerp.service.web_services.start_web_services()
     openerp.modules.module.initialize_sys_path()
     openerp.modules.loading.open_openerp_namespace()
+    openerp.pooler.get_db_and_pool('xx', update_module=False, pooljobs=False)
     for m in openerp.conf.server_wide_modules:
         try:
             __import__(m)