[MERGE] service/http: merged some mixed HTTPd changes by xrg, fixes pending

bzr revid: odo@openerp.com-20100809180832-n5ft8a8q2ixm99f2
This commit is contained in:
P. Christeas 2010-08-09 20:08:32 +02:00 committed by Olivier Dony
commit 077e387bb6
8 changed files with 297 additions and 68 deletions

View File

@ -323,6 +323,7 @@ class Server:
"""
__is_started = False
__servers = []
__starter_threads = []
# we don't want blocking server calls (think select()) to
# wait forever and possibly prevent exiting the process,
@ -335,12 +336,26 @@ class Server:
__logger = logging.getLogger('server')
def __init__(self):
if Server.__is_started:
raise Exception('All instances of servers must be inited before the startAll()')
Server.__servers.append(self)
if Server.__is_started:
# raise Exception('All instances of servers must be inited before the startAll()')
# Since the startAll() won't be called again, allow this server to
# init and then start it after 1sec (hopefully). Register that
# timer thread in a list, so that we can abort the start if quitAll
# is called in the meantime
t = threading.Timer(1.0, self._late_start)
t.name = 'Late start timer for %s' % str(self.__class__)
Server.__starter_threads.append(t)
t.start()
def start(self):
self.__logger.debug("called stub Server.start")
def _late_start(self):
self.start()
for thr in Server.__starter_threads:
if thr.finished.is_set():
Server.__starter_threads.remove(thr)
def stop(self):
self.__logger.debug("called stub Server.stop")
@ -363,6 +378,11 @@ class Server:
if not cls.__is_started:
return
cls.__logger.info("Stopping %d services" % len(cls.__servers))
for thr in cls.__starter_threads:
if not thr.finished.is_set():
thr.cancel()
cls.__starter_threads.remove(thr)
for srv in cls.__servers:
srv.stop()
cls.__is_started = False

View File

@ -108,6 +108,7 @@ if not ( tools.config["stop_after_init"] or \
tools.config["translate_out"] ):
service.http_server.init_servers()
service.http_server.init_xmlrpc()
service.http_server.init_static_http()
import service.netrpc_server
service.netrpc_server.init_servers()

View File

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
#
# Copyright P. Christeas <p_christ@hol.gr> 2008,2009
# Copyright P. Christeas <p_christ@hol.gr> 2008-2010
# Copyright 2010 OpenERP SA. (http://www.openerp.com)
#
#
# WARNING: This program as such is intended to be used by professional
@ -34,10 +35,13 @@ import netsvc
import errno
import threading
import tools
import posixpath
import urllib
import os
import select
import socket
import xmlrpclib
import logging
from SimpleXMLRPCServer import SimpleXMLRPCDispatcher
@ -69,6 +73,9 @@ class ThreadedHTTPServer(ConnThreadingMixIn, SimpleXMLRPCDispatcher, HTTPServer)
SimpleXMLRPCDispatcher.__init__(self, allow_none, encoding)
HTTPServer.__init__(self, addr, requestHandler)
self.numThreads = 0
self.__threadno = 0
# [Bug #1222790] If possible, set close-on-exec flag; if a
# method spawns a subprocess, the subprocess shouldn't have
@ -81,21 +88,44 @@ class ThreadedHTTPServer(ConnThreadingMixIn, SimpleXMLRPCDispatcher, HTTPServer)
def handle_error(self, request, client_address):
""" Override the error handler
"""
import traceback
netsvc.Logger().notifyChannel("init", netsvc.LOG_ERROR,"Server error in request from %s:\n%s" %
(client_address,traceback.format_exc()))
logging.getLogger("init").exception("Server error in request from %s:" % (client_address,))
class MultiHandler2(MultiHTTPHandler):
def _mark_start(self, thread):
self.numThreads += 1
def _mark_end(self, thread):
self.numThreads -= 1
def _get_next_name(self):
self.__threadno += 1
return 'http-client-%d' % self.__threadno
class HttpLogHandler:
""" helper class for uniform log handling
Please define self._logger at each class that is derived from this
"""
_logger = None
def log_message(self, format, *args):
netsvc.Logger().notifyChannel('http',netsvc.LOG_DEBUG,format % args)
self._logger.debug(format % args) # todo: perhaps other level
def log_error(self, format, *args):
netsvc.Logger().notifyChannel('http',netsvc.LOG_ERROR,format % args)
self._logger.error(format % args)
def log_exception(self, format, *args):
self._logger.exception(format, *args)
def log_request(self, code='-', size='-'):
self._logger.log(netsvc.logging.DEBUG_RPC, '"%s" %s %s',
self.requestline, str(code), str(size))
class MultiHandler2(HttpLogHandler, MultiHTTPHandler):
_logger = logging.getLogger('http')
class SecureMultiHandler2(SecureMultiHTTPHandler):
def log_message(self, format, *args):
netsvc.Logger().notifyChannel('https',netsvc.LOG_DEBUG,format % args)
class SecureMultiHandler2(HttpLogHandler, SecureMultiHTTPHandler):
_logger = logging.getLogger('https')
def getcert_fnames(self):
tc = tools.config
@ -103,13 +133,9 @@ class SecureMultiHandler2(SecureMultiHTTPHandler):
fkey = tc.get_misc('httpsd','sslkey', 'ssl/server.key')
return (fcert,fkey)
def log_message(self, format, *args):
netsvc.Logger().notifyChannel('http',netsvc.LOG_DEBUG,format % args)
def log_error(self, format, *args):
netsvc.Logger().notifyChannel('http',netsvc.LOG_ERROR,format % args)
class BaseHttpDaemon(threading.Thread, netsvc.Server):
_RealProto = '??'
def __init__(self, interface, port, handler):
threading.Thread.__init__(self)
netsvc.Server.__init__(self)
@ -121,10 +147,11 @@ class BaseHttpDaemon(threading.Thread, netsvc.Server):
self.server.vdirs = []
self.server.logRequests = True
self.server.timeout = self._busywait_timeout
logging.getLogger("web-services").info(
"starting %s service at %s port %d" %
(self._RealProto, interface or '0.0.0.0', port,))
except Exception, e:
netsvc.Logger().notifyChannel(
'httpd', netsvc.LOG_CRITICAL,
"Error occur when starting the server daemon: %s" % (e,))
logging.getLogger("httpd").exception("Error occured when starting the server daemon.")
raise
@property
@ -148,29 +175,51 @@ class BaseHttpDaemon(threading.Thread, netsvc.Server):
raise
return True
def stats(self):
res = "%sd: " % self._RealProto + ((self.running and "running") or "stopped")
if self.server:
res += ", %d threads" % (self.server.numThreads,)
return res
def append_svc(self, service):
if not isinstance(service, HTTPDir):
raise Exception("Wrong class for http service")
pos = len(self.server.vdirs)
lastpos = pos
while pos > 0:
pos -= 1
if self.server.vdirs[pos].matches(service.path):
lastpos = pos
# we won't break here, but search all way to the top, to
# ensure there is no lesser entry that will shadow the one
# we are inserting.
self.server.vdirs.insert(lastpos, service)
def list_services(self):
ret = []
for svc in self.server.vdirs:
ret.append( ( svc.path, str(svc.handler)) )
return ret
class HttpDaemon(BaseHttpDaemon):
_RealProto = 'HTTP'
def __init__(self, interface, port):
super(HttpDaemon, self).__init__(interface, port,
handler=MultiHandler2)
netsvc.Logger().notifyChannel(
"web-services", netsvc.LOG_INFO,
"starting HTTP service at %s port %d" %
(interface or '0.0.0.0', port,))
class HttpSDaemon(BaseHttpDaemon):
_RealProto = 'HTTPS'
def __init__(self, interface, port):
try:
super(HttpSDaemon, self).__init__(interface, port,
handler=SecureMultiHandler2)
except SSLError, e:
netsvc.Logger().notifyChannel(
'httpd-ssl', netsvc.LOG_CRITICAL,
"Can not load the certificate and/or the private key files")
logging.getLogger('httpsd').exception( \
"Can not load the certificate and/or the private key files")
raise
netsvc.Logger().notifyChannel(
"web-services", netsvc.LOG_INFO,
"starting HTTPS service at %s port %d" %
(interface or '0.0.0.0', port,))
httpd = None
httpsd = None
@ -190,23 +239,32 @@ def reg_http_service(hts, secure_only = False):
hts must be an HTTPDir
"""
global httpd, httpsd
if not isinstance(hts, HTTPDir):
raise Exception("Wrong class for http service")
if httpd and not secure_only:
httpd.server.vdirs.append(hts)
httpd.append_svc(hts)
if httpsd:
httpsd.server.vdirs.append(hts)
httpsd.append_svc(hts)
if (not httpd) and (not httpsd):
netsvc.Logger().notifyChannel('httpd',netsvc.LOG_WARNING,"No httpd available to register service %s" % hts.path)
logging.getLogger('httpd').warning("No httpd available to register service %s" % hts.path)
return
def list_http_services(protocol=None):
global httpd, httpsd
if httpd and (protocol == 'http' or protocol == None):
return httpd.list_services()
elif httpsd and (protocol == 'https' or protocol == None):
return httpsd.list_services()
else:
raise Exception("Incorrect protocol or no http services")
import SimpleXMLRPCServer
class XMLRPCRequestHandler(netsvc.OpenERPDispatcher,FixSendError,SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
class XMLRPCRequestHandler(netsvc.OpenERPDispatcher,FixSendError,HttpLogHandler,SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
rpc_paths = []
protocol_version = 'HTTP/1.1'
_logger = logging.getLogger('xmlrpc')
def _dispatch(self, method, params):
try:
service_name = self.path.split("/")[-1]
@ -214,9 +272,6 @@ class XMLRPCRequestHandler(netsvc.OpenERPDispatcher,FixSendError,SimpleXMLRPCSer
except netsvc.OpenERPDispatcherException, e:
raise xmlrpclib.Fault(tools.exception_to_unicode(e.exception), e.traceback)
def log_message(self, format, *args):
netsvc.Logger().notifyChannel('xmlrpc',netsvc.LOG_DEBUG_RPC,format % args)
def handle(self):
pass
@ -235,13 +290,57 @@ def init_xmlrpc():
# Example of http file serving:
# reg_http_service(HTTPDir('/test/',HTTPHandler))
reg_http_service(HTTPDir('/xmlrpc/', XMLRPCRequestHandler))
netsvc.Logger().notifyChannel("web-services", netsvc.LOG_INFO,
"Registered XML-RPC over HTTP")
logging.getLogger("web-services").info("Registered XML-RPC over HTTP")
if tools.config.get('xmlrpcs', False):
reg_http_service(HTTPDir('/xmlrpc/', XMLRPCRequestHandler, True))
netsvc.Logger().notifyChannel('web-services', netsvc.LOG_INFO,
"Registered XML-RPC over HTTPS")
if tools.config.get('xmlrpcs', False) \
and not tools.config.get('xmlrpc', False):
# only register at the secure server
reg_http_service(HTTPDir('/xmlrpc/', XMLRPCRequestHandler), True)
logging.getLogger("web-services").info("Registered XML-RPC over HTTPS only")
class StaticHTTPHandler(HttpLogHandler, FixSendError, HttpOptions, HTTPHandler):
_logger = logging.getLogger('httpd')
_HTTP_OPTIONS = { 'Allow': ['OPTIONS', 'GET', 'HEAD'] }
def __init__(self,request, client_address, server):
HTTPHandler.__init__(self,request,client_address,server)
dir_path = tools.config.get_misc('static-http', 'dir_path', False)
assert dir_path, "Please specify static-http/dir_path in config, or disable static-httpd!"
self.__basepath = dir_path
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
"""
# abandon query parameters
path = path.split('?',1)[0]
path = path.split('#',1)[0]
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = self.__basepath
for word in words:
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def init_static_http():
if not tools.config.get_misc('static-http','enable', False):
return
dir_path = tools.config.get_misc('static-http', 'dir_path', False)
assert dir_path
base_path = tools.config.get_misc('static-http', 'base_path', '/')
reg_http_service(HTTPDir(base_path,StaticHTTPHandler))
logging.getLogger("web-services").info("Registered HTTP dir %s for %s" % \
(dir_path, base_path))
class OerpAuthProxy(AuthProxy):
""" Require basic authentication..
@ -260,7 +359,6 @@ class OerpAuthProxy(AuthProxy):
try:
if not db:
db = handler.get_db_from_path(path)
print "Got db:",db
except Exception:
if path.startswith('/'):
path = path[1:]
@ -305,10 +403,10 @@ class OpenERPAuthProvider(AuthProvider):
return False
return (user, passwd, db, uid)
except Exception,e:
netsvc.Logger().notifyChannel("auth",netsvc.LOG_DEBUG,"Fail auth:"+ str(e))
logging.getLogger("auth").debug("Fail auth: %s" % e )
return False
def log(self, msg):
netsvc.Logger().notifyChannel("auth",netsvc.LOG_INFO,msg)
def log(self, msg, lvl=logging.INFO):
logging.getLogger("auth").log(lvl,msg)
#eof

View File

@ -36,7 +36,9 @@ import tools
class TinySocketClientThread(threading.Thread, netsvc.OpenERPDispatcher):
def __init__(self, sock, threads):
threading.Thread.__init__(self)
spn = sock and sock.getpeername()
spn = 'netrpc-client-%s:%s' % spn[0:2]
threading.Thread.__init__(self, name=spn)
self.sock = sock
# Only at the server side, use a big timeout: close the
# clients connection when they're idle for 20min.
@ -48,7 +50,8 @@ class TinySocketClientThread(threading.Thread, netsvc.OpenERPDispatcher):
try:
self.socket.shutdown(
getattr(socket, 'SHUT_RDWR', 2))
except Exception: pass
except Exception:
pass
# That should garbage-collect and close it, too
self.sock = None

View File

@ -305,7 +305,7 @@ class db(netsvc.ExportService):
else:
cr.execute("select decode(datname, 'escape') from pg_database where datname not in('template0', 'template1','postgres') order by datname")
res = [str(name) for (name,) in cr.fetchall()]
except:
except Exception:
res = []
finally:
cr.close()
@ -385,7 +385,8 @@ class common(_ObjectService):
logger.notifyChannel("web-service", netsvc.LOG_INFO,'Logout %s from database %s'%(login,db))
return True
elif method in ['about', 'timezone_get', 'get_server_environment',
'login_message','get_stats', 'check_connectivity']:
'login_message','get_stats', 'check_connectivity',
'list_http_services']:
pass
elif method in ['get_available_updates', 'get_migration_scripts', 'set_loglevel']:
passwd = params[0]
@ -492,7 +493,7 @@ GNU Public Licence.
try:
try:
base64_decoded = base64.decodestring(zips[module])
except:
except Exception:
l.notifyChannel('migration', netsvc.LOG_ERROR, 'unable to read the module %s' % (module,))
raise
@ -501,13 +502,13 @@ GNU Public Licence.
try:
try:
tools.extract_zip_file(zip_contents, tools.config['addons_path'] )
except:
except Exception:
l.notifyChannel('migration', netsvc.LOG_ERROR, 'unable to extract the module %s' % (module, ))
rmtree(module)
raise
finally:
zip_contents.close()
except:
except Exception:
l.notifyChannel('migration', netsvc.LOG_ERROR, 'restore the previous version of the module %s' % (module, ))
nmp = os.path.join(backup_directory, module)
if os.path.isdir(nmp):
@ -563,6 +564,10 @@ GNU Public Licence.
res += netsvc.Server.allStats()
return res
def exp_list_http_services(self):
from service import http_server
return http_server.list_http_services()
def exp_check_connectivity(self):
return bool(sql_db.db_connect('template1'))

View File

@ -142,6 +142,9 @@ class noconnection:
def makefile(self, mode, bufsize):
return None
def close(self):
pass
class dummyconn:
def shutdown(self, tru):
pass
@ -169,10 +172,45 @@ class FixSendError:
self.send_header('Connection', 'close')
self.send_header('Content-Length', len(content) or 0)
self.end_headers()
if hasattr(self, '_flush'):
self._flush()
if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
self.wfile.write(content)
class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
class HttpOptions:
_HTTP_OPTIONS = {'Allow': ['OPTIONS' ] }
def do_OPTIONS(self):
"""return the list of capabilities """
opts = self._HTTP_OPTIONS
nopts = self._prep_OPTIONS(opts)
if nopts:
opts = nopts
self.send_response(200)
self.send_header("Content-Length", 0)
for key, value in opts.items():
if isinstance(value, basestring):
self.send_header(key, value)
elif isinstance(value, (tuple, list)):
self.send_header(key, ', '.join(value))
self.end_headers()
def _prep_OPTIONS(self, opts):
"""Prepare the OPTIONS response, if needed
Sometimes, like in special DAV folders, the OPTIONS may contain
extra keywords, perhaps also dependant on the request url.
@param the options already. MUST be copied before being altered
@return the updated options.
"""
return opts
class MultiHTTPHandler(FixSendError, HttpOptions, BaseHTTPRequestHandler):
""" this is a multiple handler, that will dispatch each request
to a nested handler, iff it matches
@ -226,11 +264,30 @@ class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
return
mname = 'do_' + fore.command
if not hasattr(fore, mname):
fore.send_error(501, "Unsupported method (%r)" % fore.command)
if fore.command == 'OPTIONS':
self.do_OPTIONS()
return
self.send_error(501, "Unsupported method (%r)" % fore.command)
return
fore.close_connection = 0
method = getattr(fore, mname)
method()
try:
method()
except (AuthRejectedExc, AuthRequiredExc):
raise
except Exception, e:
if hasattr(self, 'log_exception'):
self.log_exception("Could not run %s", mname)
else:
self.log_error("Could not run %s: %s", mname, e)
self.send_error(500, "Internal error")
# may not work if method has already sent data
fore.close_connection = 1
self.close_connection = 1
if hasattr(fore, '_flush'):
fore._flush()
return
if fore.close_connection:
# print "Closing connection because of handler"
self.close_connection = fore.close_connection
@ -289,6 +346,7 @@ class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
[command, path] = words
self.close_connection = 1
if command != 'GET':
self.log_error("Junk http request: %s", self.raw_requestline)
self.send_error(400,
"Bad HTTP/0.9 request type (%r)" % command)
return False
@ -315,6 +373,14 @@ class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
self.log_message("Could not parse rawline.")
return
# self.parse_request(): # Do NOT parse here. the first line should be the only
if self.path == '*' and self.command == 'OPTIONS':
# special handling of path='*', must not use any vdir at all.
if not self.parse_request():
return
self.do_OPTIONS()
return
for vdir in self.server.vdirs:
p = vdir.matches(self.path)
if p == False:
@ -397,13 +463,26 @@ class ConnThreadingMixIn:
# main process
daemon_threads = False
def _get_next_name(self):
return None
def _handle_request_noblock(self):
"""Start a new thread to process the request."""
t = threading.Thread(target = self._handle_request2)
if not threading: # happens while quitting python
return
t = threading.Thread(name=self._get_next_name(), target=self._handle_request2)
if self.daemon_threads:
t.setDaemon (1)
t.start()
def _mark_start(self, thread):
""" Mark the start of a request thread """
pass
def _mark_end(self, thread):
""" Mark the end of a request thread """
pass
def _handle_request2(self):
"""Handle one request, without blocking.
@ -412,14 +491,17 @@ class ConnThreadingMixIn:
no risk of blocking in get_request().
"""
try:
self._mark_start(threading.currentThread())
request, client_address = self.get_request()
if self.verify_request(request, client_address):
try:
self.process_request(request, client_address)
except Exception:
self.handle_error(request, client_address)
self.close_request(request)
except socket.error:
return
if self.verify_request(request, client_address):
try:
self.process_request(request, client_address)
except Exception:
self.handle_error(request, client_address)
self.close_request(request)
finally:
self._mark_end(threading.currentThread())
#eof

View File

@ -25,3 +25,7 @@ translate_modules = ['all']
demo = {}
addons_path = None
reportgz = False
[static-http]
enable = False
dir_path = /var/www/html

16
list-services.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/bash
# ADMIN_PASSWD='admin'
method_1() {
cat '-' << EOF
<xml>
<methodCall>
<methodName>list_http_services</methodName>
<params>
</params>
</methodCall>
EOF
}
method_1 | POST -c 'text/xml' http://localhost:8069/xmlrpc/common
#eof