From ba178be7c0828dc0df33b0d22347cc9e92dbb574 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 20 Jan 2012 12:46:12 +0100 Subject: [PATCH 1/4] [IMP] gunicorn: add CPU and memory limits. bzr revid: vmt@openerp.com-20120120114612-xowu57yy3f5uxi0j --- gunicorn.conf.py | 34 ++++++++++++++- openerp/tests/addons/test_limits/__init__.py | 3 ++ .../tests/addons/test_limits/__openerp__.py | 15 +++++++ openerp/tests/addons/test_limits/models.py | 43 +++++++++++++++++++ openerp/wsgi.py | 1 + 5 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 openerp/tests/addons/test_limits/__init__.py create mode 100644 openerp/tests/addons/test_limits/__openerp__.py create mode 100644 openerp/tests/addons/test_limits/models.py diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 9962493d549..c575d53875f 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -17,7 +17,7 @@ pidfile = '.gunicorn.pid' # Gunicorn recommends 2-4 x number_of_cpu_cores, but # you'll want to vary this a bit to find the best for your # particular work load. -workers = 4 +workers = 1 # Some application-wide initialization is needed. on_starting = openerp.wsgi.on_starting @@ -27,6 +27,8 @@ when_ready = openerp.wsgi.when_ready # big reports for example timeout = 240 +#max_requests = 150 + # Equivalent of --load command-line option openerp.conf.server_wide_modules = ['web'] @@ -35,7 +37,7 @@ conf = openerp.tools.config # Path to the OpenERP Addons repository (comma-separated for # multiple locations) -conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons' +conf['addons_path'] = '/home/thu/repos/addons/trunk,/home/thu/repos/web/trunk/addons,/home/thu/repos/server/trunk-limits/openerp/tests/addons' # Optional database config if not using local socket #conf['db_name'] = 'mycompany' @@ -52,5 +54,33 @@ conf['addons_path'] = '/home/openerp/addons/trunk,/home/openerp/web/trunk/addons # If --static-http-enable is used, path for the static web directory #conf['static_http_document_root'] = '/var/www' +def time_expired(n, stack): + import os + import time + print '>>> [%s] time ran out.' % (os.getpid()) + raise Exception('(time ran out)') + +def pre_request(worker, req): + import os + import psutil + import resource + import signal + # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space + rss, vms = psutil.Process(os.getpid()).get_memory_info() + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment) + print ">>>>>> %s" % (req.body,) + # Let's say 512MB is ok, 768 is a hard limit (will raise MemoryError), + # 640 will nicely restart the process. + resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, hard)) + if vms > 640 * 1024 * 1024: + print ">>> Worker eating too much memory, reset it after the request." + worker.alive = False # Commit suicide after the request. + + r = resource.getrusage(resource.RUSAGE_SELF) + cpu_time = r.ru_utime + r.ru_stime + signal.signal(signal.SIGXCPU, time_expired) + soft, hard = resource.getrlimit(resource.RLIMIT_CPU) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + 15, hard)) # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/__init__.py b/openerp/tests/addons/test_limits/__init__.py new file mode 100644 index 00000000000..fe4487156b1 --- /dev/null +++ b/openerp/tests/addons/test_limits/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- +import models +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/__openerp__.py b/openerp/tests/addons/test_limits/__openerp__.py new file mode 100644 index 00000000000..333f1fe4372 --- /dev/null +++ b/openerp/tests/addons/test_limits/__openerp__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'test-limits', + 'version': '0.1', + 'category': 'Tests', + 'description': """A module with dummy methods.""", + 'author': 'OpenERP SA', + 'maintainer': 'OpenERP SA', + 'website': 'http://www.openerp.com', + 'depends': ['base'], + 'data': [], + 'installable': True, + 'active': False, +} +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tests/addons/test_limits/models.py b/openerp/tests/addons/test_limits/models.py new file mode 100644 index 00000000000..435e93811fc --- /dev/null +++ b/openerp/tests/addons/test_limits/models.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +import time + +import openerp + +class m(openerp.osv.osv.Model): + """ This model exposes a few methods that will consume between 'almost no + resource' and 'a lot of resource'. + """ + _name = 'test.limits.model' + + def consume_nothing(self, cr, uid, context=None): + return True + + def consume_memory(self, cr, uid, size, context=None): + l = [0] * size + return True + + def leak_memory(self, cr, uid, size, context=None): + if not hasattr(self, 'l'): + self.l = [] + self.l.append([0] * size) + print ">>>", len(self.l) + return True + + def consume_time(self, cr, uid, seconds, context=None): + time.sleep(seconds) + return True + + def consume_cpu_time(self, cr, uid, seconds, context=None): + import os + t0 = time.clock() + t1 = time.clock() +# try: + while t1 - t0 < seconds: + print "[%s] ..." % os.getpid() + for i in xrange(10000000): + x = i * i + t1 = time.clock() +# except Exception, e: +# print "+++", e + return True +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/wsgi.py b/openerp/wsgi.py index aeba38dcfbc..b9263b42352 100644 --- a/openerp/wsgi.py +++ b/openerp/wsgi.py @@ -460,6 +460,7 @@ def on_starting(server): openerp.service.web_services.start_web_services() openerp.modules.module.initialize_sys_path() openerp.modules.loading.open_openerp_namespace() + openerp.pooler.get_db_and_pool('xx', update_module=False, pooljobs=False) for m in openerp.conf.server_wide_modules: try: __import__(m) From 09347af434e8ebe1ad828c44703ec8d8ab6870d8 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 20 Jan 2012 16:00:50 +0100 Subject: [PATCH 2/4] [IMP] gunicorn: moved gunicorn hook to openerp.wsgi (just like previous hooks), added new command-line options. bzr revid: vmt@openerp.com-20120120150050-3o3hg6k1n17alup0 --- gunicorn.conf.py | 32 ++------------------------------ openerp/tools/config.py | 17 ++++++++++++++++- openerp/wsgi.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 31 deletions(-) diff --git a/gunicorn.conf.py b/gunicorn.conf.py index c575d53875f..05feb71ada8 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -22,12 +22,13 @@ workers = 1 # Some application-wide initialization is needed. on_starting = openerp.wsgi.on_starting when_ready = openerp.wsgi.when_ready +pre_request = openerp.wsgi.pre_request # openerp request-response cycle can be quite long for # big reports for example timeout = 240 -#max_requests = 150 +max_requests = 2000 # Equivalent of --load command-line option openerp.conf.server_wide_modules = ['web'] @@ -54,33 +55,4 @@ conf['addons_path'] = '/home/thu/repos/addons/trunk,/home/thu/repos/web/trunk/ad # If --static-http-enable is used, path for the static web directory #conf['static_http_document_root'] = '/var/www' -def time_expired(n, stack): - import os - import time - print '>>> [%s] time ran out.' % (os.getpid()) - raise Exception('(time ran out)') - -def pre_request(worker, req): - import os - import psutil - import resource - import signal - # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space - rss, vms = psutil.Process(os.getpid()).get_memory_info() - soft, hard = resource.getrlimit(resource.RLIMIT_AS) - print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment) - print ">>>>>> %s" % (req.body,) - # Let's say 512MB is ok, 768 is a hard limit (will raise MemoryError), - # 640 will nicely restart the process. - resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, hard)) - if vms > 640 * 1024 * 1024: - print ">>> Worker eating too much memory, reset it after the request." - worker.alive = False # Commit suicide after the request. - - r = resource.getrusage(resource.RUSAGE_SELF) - cpu_time = r.ru_utime + r.ru_stime - signal.signal(signal.SIGXCPU, time_expired) - soft, hard = resource.getrlimit(resource.RLIMIT_CPU) - resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + 15, hard)) - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tools/config.py b/openerp/tools/config.py index 8107e10bf85..4f33c3b79f1 100644 --- a/openerp/tools/config.py +++ b/openerp/tools/config.py @@ -266,6 +266,20 @@ class configmanager(object): group.add_option("--max-cron-threads", dest="max_cron_threads", my_default=4, help="Maximum number of threads processing concurrently cron jobs.", type="int") + # TODO sensible default for the three following limits. + group.add_option("--virtual-memory-limit", dest="virtual_memory_limit", my_default=768 * 1024 * 1024, + help="Maximum allowed virtual memory per Gunicorn process. " + "When the limit is reached, any memory allocation will fail.", + type="int") + group.add_option("--virtual-memory-reset", dest="virtual_memory_reset", my_default=640 * 1024 * 1024, + help="Maximum allowed virtual memory per Gunicorn process. " + "When the limit is reached, the worker will be reset after " + "the current request.", + type="int") + group.add_option("--cpu-time-limit", dest="cpu_time_limit", my_default=60, + help="Maximum allowed CPU time per Gunicorn process. " + "When the limit is reached, an exception is raised.", + type="int") group.add_option("--unaccent", dest="unaccent", my_default=False, action="store_true", help="Use the unaccent function provided by the database when available.") @@ -371,7 +385,8 @@ class configmanager(object): 'stop_after_init', 'logrotate', 'without_demo', 'netrpc', 'xmlrpc', 'syslog', 'list_db', 'xmlrpcs', 'test_file', 'test_disable', 'test_commit', 'test_report_directory', - 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', 'unaccent', + 'osv_memory_count_limit', 'osv_memory_age_limit', 'max_cron_threads', + 'virtual_memory_limit', 'virtual_memory_reset', 'cpu_time_limit', 'unaccent', ] for arg in keys: diff --git a/openerp/wsgi.py b/openerp/wsgi.py index b9263b42352..88c75539dc2 100644 --- a/openerp/wsgi.py +++ b/openerp/wsgi.py @@ -481,12 +481,41 @@ def when_ready(server): # Hijack gunicorn's SIGWINCH handling; we can choose another one. signal.signal(signal.SIGWINCH, make_winch_handler(server)) +# Install limits on virtual memory and CPU time consumption. +def pre_request(worker, req): + import os + import psutil + import resource + import signal + # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space + rss, vms = psutil.Process(os.getpid()).get_memory_info() + soft, hard = resource.getrlimit(resource.RLIMIT_AS) + print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment) + print ">>>>>> %s" % (req.body,) + resource.setrlimit(resource.RLIMIT_AS, (config['virtual_memory_limit'], hard)) + if vms > config['virtual_memory_reset']: + print ">>> Worker eating too much memory, reset it after the request." + worker.alive = False # Commit suicide after the request. + + r = resource.getrusage(resource.RUSAGE_SELF) + cpu_time = r.ru_utime + r.ru_stime + signal.signal(signal.SIGXCPU, time_expired) + soft, hard = resource.getrlimit(resource.RLIMIT_CPU) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['cpu_time_limit'], hard)) + # Our signal handler will signal a SGIQUIT to all workers. def make_winch_handler(server): def handle_winch(sig, fram): server.kill_workers(signal.SIGQUIT) # This is gunicorn specific. return handle_winch +# SIGXCPU (exceeded CPU time) signal handler will raise an exception. +def time_expired(n, stack): + import os + import time + print '>>> [%s] time ran out.' % (os.getpid()) + raise Exception('(time ran out)') # TODO one of openerp.exception + # Kill gracefuly the workers (e.g. because we want to clear their cache). # This is done by signaling a SIGWINCH to the master process, so it can be # called by the workers themselves. From 5e2721aaf247a75f8361f1272617b2cf7c27bafd Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 20 Jan 2012 16:43:22 +0100 Subject: [PATCH 3/4] [IMP] gunicorn: commit suicide in the post_request hook instead of the pre_request hook. bzr revid: vmt@openerp.com-20120120154322-f23rxofv0169tbsm --- gunicorn.conf.py | 1 + openerp/wsgi.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 05feb71ada8..4d85fa2b6c0 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -23,6 +23,7 @@ workers = 1 on_starting = openerp.wsgi.on_starting when_ready = openerp.wsgi.when_ready pre_request = openerp.wsgi.pre_request +post_request = openerp.wsgi.post_request # openerp request-response cycle can be quite long for # big reports for example diff --git a/openerp/wsgi.py b/openerp/wsgi.py index 88c75539dc2..ed03ed9bc6c 100644 --- a/openerp/wsgi.py +++ b/openerp/wsgi.py @@ -491,11 +491,7 @@ def pre_request(worker, req): rss, vms = psutil.Process(os.getpid()).get_memory_info() soft, hard = resource.getrlimit(resource.RLIMIT_AS) print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment) - print ">>>>>> %s" % (req.body,) resource.setrlimit(resource.RLIMIT_AS, (config['virtual_memory_limit'], hard)) - if vms > config['virtual_memory_reset']: - print ">>> Worker eating too much memory, reset it after the request." - worker.alive = False # Commit suicide after the request. r = resource.getrusage(resource.RUSAGE_SELF) cpu_time = r.ru_utime + r.ru_stime @@ -503,6 +499,15 @@ def pre_request(worker, req): soft, hard = resource.getrlimit(resource.RLIMIT_CPU) resource.setrlimit(resource.RLIMIT_CPU, (cpu_time + config['cpu_time_limit'], hard)) +# Reset the worker if it consumes too much memory (e.g. caused by a memory leak). +def post_request(worker, req, environ): + import os + import psutil + rss, vms = psutil.Process(os.getpid()).get_memory_info() + if vms > config['virtual_memory_reset']: + print ">>> Worker eating too much memory, reset it after the request." + worker.alive = False # Commit suicide after the request. + # Our signal handler will signal a SGIQUIT to all workers. def make_winch_handler(server): def handle_winch(sig, fram): From 30d7253fef235bbb117aeec0a27c7c5e3c436c4a Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Fri, 20 Jan 2012 17:04:09 +0100 Subject: [PATCH 4/4] [IMP] gunicorn: changed `print` with `logging.info`. bzr revid: vmt@openerp.com-20120120160409-cu1vcw7cfa3z0zgy --- gunicorn.conf.py | 2 +- openerp/tests/addons/test_limits/models.py | 5 ----- openerp/wsgi.py | 10 ++++------ 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/gunicorn.conf.py b/gunicorn.conf.py index 4d85fa2b6c0..db570401d17 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -17,7 +17,7 @@ pidfile = '.gunicorn.pid' # Gunicorn recommends 2-4 x number_of_cpu_cores, but # you'll want to vary this a bit to find the best for your # particular work load. -workers = 1 +workers = 4 # Some application-wide initialization is needed. on_starting = openerp.wsgi.on_starting diff --git a/openerp/tests/addons/test_limits/models.py b/openerp/tests/addons/test_limits/models.py index 435e93811fc..5240acd23ab 100644 --- a/openerp/tests/addons/test_limits/models.py +++ b/openerp/tests/addons/test_limits/models.py @@ -20,7 +20,6 @@ class m(openerp.osv.osv.Model): if not hasattr(self, 'l'): self.l = [] self.l.append([0] * size) - print ">>>", len(self.l) return True def consume_time(self, cr, uid, seconds, context=None): @@ -31,13 +30,9 @@ class m(openerp.osv.osv.Model): import os t0 = time.clock() t1 = time.clock() -# try: while t1 - t0 < seconds: - print "[%s] ..." % os.getpid() for i in xrange(10000000): x = i * i t1 = time.clock() -# except Exception, e: -# print "+++", e return True # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/wsgi.py b/openerp/wsgi.py index ed03ed9bc6c..7d47cdb7bb6 100644 --- a/openerp/wsgi.py +++ b/openerp/wsgi.py @@ -490,7 +490,6 @@ def pre_request(worker, req): # VMS and RLIMIT_AS are the same thing: virtual memory, a.k.a. address space rss, vms = psutil.Process(os.getpid()).get_memory_info() soft, hard = resource.getrlimit(resource.RLIMIT_AS) - print ">>>>>> [%s] %s %s %s %s %s" % (os.getpid(), vms, req.method, req.path, req.query, req.fragment) resource.setrlimit(resource.RLIMIT_AS, (config['virtual_memory_limit'], hard)) r = resource.getrusage(resource.RUSAGE_SELF) @@ -505,7 +504,8 @@ def post_request(worker, req, environ): import psutil rss, vms = psutil.Process(os.getpid()).get_memory_info() if vms > config['virtual_memory_reset']: - print ">>> Worker eating too much memory, reset it after the request." + logging.getLogger('wsgi.worker').info('Virtual memory consumption ' + 'too high, rebooting the worker.') worker.alive = False # Commit suicide after the request. # Our signal handler will signal a SGIQUIT to all workers. @@ -516,10 +516,8 @@ def make_winch_handler(server): # SIGXCPU (exceeded CPU time) signal handler will raise an exception. def time_expired(n, stack): - import os - import time - print '>>> [%s] time ran out.' % (os.getpid()) - raise Exception('(time ran out)') # TODO one of openerp.exception + logging.getLogger('wsgi.worker').info('CPU time limit exceeded.') + raise Exception('CPU time limit exceeded.') # TODO one of openerp.exception # Kill gracefuly the workers (e.g. because we want to clear their cache). # This is done by signaling a SIGWINCH to the master process, so it can be