From ba178be7c0828dc0df33b0d22347cc9e92dbb574 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 20 Jan 2012 12:46:12 +0100 Subject: [PATCH] [IMP] gunicorn: add CPU and memory limits. bzr revid: vmt@openerp.com-20120120114612-xowu57yy3f5uxi0j --- gunicorn.conf.py | 34 ++++++++++++++- openerp/tests/addons/test_limits/__init__.py | 3 ++ .../tests/addons/test_limits/__openerp__.py | 15 +++++++ openerp/tests/addons/test_limits/models.py | 43 +++++++++++++++++++ openerp/wsgi.py | 1 + 5 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 openerp/tests/addons/test_limits/__init__.py create mode 100644 openerp/tests/addons/test_limits/__openerp__.py create mode 100644 openerp/tests/addons/test_limits/models.py diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 9962493d549..c575d53875f 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -17,7 +17,7 @@ pidfile = '.gunicorn.pid' # Gunicorn recommends 2-4 x number_of_cpu_cores, but # you'll want to vary this a bit to find the best for your # particular work load. -workers = 4 +workers = 1 # Some application-wide initialization is needed. on_starting = openerp.wsgi.on_starting @@ -27,6 +27,8 @@ when_ready = openerp.wsgi.when_ready # big reports for example timeout = 240 +#max_requests = 150 + # Equivalent of --load command-line option openerp.conf.server_wide_modules = ['web'] @@ -35,7 +37,7 @@ conf = openerp.tools.config # Path to the OpenERP Addons repository (comma-separated for # multiple locations) -conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons' +conf['addons_path'] = '/home/thu/repos/addons/trunk,/home/thu/repos/web/trunk/addons,/home/thu/repos/server/trunk-limits/openerp/tests/addons' # Optional database config if not using local socket #conf['db_name'] = 'mycompany' @@ -52,5 +54,33 @@ conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons # If --static-http-enable is used, path for the static web directory #conf['static_http_document_root'] = '/var/www' +def time_expired(n, stack): + import os + import time + print '>>> [%s] time ran out.' % (os.getpid()) + raise Exception('(time ran out)') + +def pre_request(worker, req): + import os + import psutil + import resource + import signal + # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space + rss, vms = psutil.Process(os.getpid()).get_memory_info() + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment) + print ">>>>>> %s" % (req.body,) + # Let's say 512MB is ok, 768 is a hard limit (will raise MemoryError), + # 640 will nicely restart the process. + resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, hard)) + if vms > 640 * 1024 * 1024: + print ">>> Worker eating too much memory, reset it after the request." + worker.alive = False # Commit suicide after the request. + + r = resource.getrusage(resource.RUSAGE_SELF) + cpu_time = r.ru_utime + r.ru_stime + signal.signal(signal.SIGXCPU, time_expired) + soft, hard = resource.getrlimit(resource.RLIMIT_CPU) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + 15, hard)) # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/__init__.py b/openerp/tests/addons/test_limits/__init__.py new file mode 100644 index 00000000000..fe4487156b1 --- /dev/null +++ b/openerp/tests/addons/test_limits/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +import models +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/__openerp__.py b/openerp/tests/addons/test_limits/__openerp__.py new file mode 100644 index 00000000000..333f1fe4372 --- /dev/null +++ b/openerp/tests/addons/test_limits/__openerp__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'test-limits', + 'version': '0.1', + 'category': 'Tests', + 'description': """A module with dummy methods.""", + 'author': 'OpenERP SA', + 'maintainer': 'OpenERP SA', + 'website': 'http://www.openerp.com', + 'depends': ['base'], + 'data': [], + 'installable': True, + 'active': False, +} +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/models.py b/openerp/tests/addons/test_limits/models.py new file mode 100644 index 00000000000..435e93811fc --- /dev/null +++ b/openerp/tests/addons/test_limits/models.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +import time + +import openerp + +class m(openerp.osv.osv.Model): + """ This model exposes a few methods that will consume between 'almost no + resource' and 'a lot of resource'. + """ + _name = 'test.limits.model' + + def consume_nothing(self, cr, uid, context=None): + return True + + def consume_memory(self, cr, uid, size, context=None): + l = [0] * size + return True + + def leak_memory(self, cr, uid, size, context=None): + if not hasattr(self, 'l'): + self.l = [] + self.l.append([0] * size) + print ">>>", len(self.l) + return True + + def consume_time(self, cr, uid, seconds, context=None): + time.sleep(seconds) + return True + + def consume_cpu_time(self, cr, uid, seconds, context=None): + import os + t0 = time.clock() + t1 = time.clock() +# try: + while t1 - t0 < seconds: + print "[%s] ..." % os.getpid() + for i in xrange(10000000): + x = i * i + t1 = time.clock() +# except Exception, e: +# print "+++", e + return True +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/wsgi.py b/openerp/wsgi.py index aeba38dcfbc..b9263b42352 100644 --- a/openerp/wsgi.py +++ b/openerp/wsgi.py @@ -460,6 +460,7 @@ def on_starting(server): openerp.service.web_services.start_web_services() openerp.modules.module.initialize_sys_path() openerp.modules.loading.open_openerp_namespace() + openerp.pooler.get_db_and_pool('xx', update_module=False, pooljobs=False) for m in openerp.conf.server_wide_modules: try: __import__(m)