From ec9a543014a91be17398ba6e443578fe37c3a885 Mon Sep 17 00:00:00 2001 From: Antony Lesuisse Date: Mon, 19 Jan 2015 01:41:08 +0100 Subject: [PATCH] [FIX] dbmanager: backup support both zip and pg_dump custom format - let the user choose between the pg_dump custom format or the zip format including the filestore - use file objects to allow dumps larger than memory - postgres subprocess invocation is now clean and thread-safe, we dont touch the local process environ anymore - add a manifest to the zip dump format with version information about odoo, postgres (pg_dump doesnt output it) and modules --- addons/web/controllers/main.py | 26 +++---- addons/web/static/src/xml/base.xml | 9 +++ openerp/service/db.py | 112 ++++++++++++++--------------- openerp/tools/misc.py | 67 ++++++++++------- 4 files changed, 118 insertions(+), 96 deletions(-) diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 1cea529a86a..8bf395dc59b 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -724,21 +724,21 @@ class Database(http.Controller): return {'error': _('Could not drop database !'), 'title': _('Drop Database')} @http.route('/web/database/backup', type='http', auth="none") - def backup(self, backup_db, backup_pwd, token): + def backup(self, backup_db, backup_pwd, token, backup_format='zip'): try: - db_dump = base64.b64decode( - request.session.proxy("db").dump(backup_pwd, backup_db)) - filename = "%(db)s_%(timestamp)s.dump" % { - 'db': backup_db, - 'timestamp': datetime.datetime.utcnow().strftime( - "%Y-%m-%d_%H-%M-%SZ") - } - return request.make_response(db_dump, - [('Content-Type', 'application/octet-stream; charset=binary'), - ('Content-Disposition', content_disposition(filename))], - {'fileToken': token} - ) + openerp.service.security.check_super(backup_pwd) + ts = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S") + filename = "%s_%s.%s" % (backup_db, ts, backup_format) + headers = [ + ('Content-Type', 'application/octet-stream; charset=binary'), + ('Content-Disposition', content_disposition(filename)), + ] + dump_stream = openerp.service.db.dump_db_stream(backup_db, backup_format) + response = werkzeug.wrappers.Response(dump_stream, headers=headers, direct_passthrough=True) + response.set_cookie('fileToken', token) + return response except Exception, e: + _logger.exception('Database.backup') return simplejson.dumps([[],[{'error': openerp.tools.ustr(e), 'title': _('Backup Database')}]]) @http.route('/web/database/restore', type='http', auth="none") diff --git a/addons/web/static/src/xml/base.xml b/addons/web/static/src/xml/base.xml index 9b80a868c6d..864bc899960 100644 --- a/addons/web/static/src/xml/base.xml +++ b/addons/web/static/src/xml/base.xml @@ -225,6 +225,15 @@ + + + + + + diff --git a/openerp/service/db.py b/openerp/service/db.py index 048c160916f..a0a4b5e2707 100644 --- a/openerp/service/db.py +++ b/openerp/service/db.py @@ -1,14 +1,17 @@ # -*- coding: utf-8 -*- -from contextlib import closing -from functools import wraps + +import json import logging import os import shutil +import tempfile import threading import traceback -import tempfile import zipfile +from functools import wraps +from contextlib import closing + import psycopg2 import openerp @@ -143,70 +146,65 @@ def exp_drop(db_name): shutil.rmtree(fs) return True -def _set_pg_password_in_environment(func): - """ On systems where pg_restore/pg_dump require an explicit - password (i.e. when not connecting via unix sockets, and most - importantly on Windows), it is necessary to pass the PG user - password in the environment or in a special .pgpass file. - - This decorator handles setting - :envvar:`PGPASSWORD` if it is not already - set, and removing it afterwards. - - See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html - - .. note:: This is not thread-safe, and should never be enabled for - SaaS (giving SaaS users the super-admin password is not a good idea - anyway) - """ - @wraps(func) - def wrapper(*args, **kwargs): - if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']: - return func(*args, **kwargs) - else: - os.environ['PGPASSWORD'] = openerp.tools.config['db_password'] - try: - return func(*args, **kwargs) - finally: - del os.environ['PGPASSWORD'] - return wrapper - def exp_dump(db_name): with tempfile.TemporaryFile() as t: dump_db(db_name, t) t.seek(0) return t.read().encode('base64') -@_set_pg_password_in_environment -def dump_db(db, stream): +def dump_db_manifest(cr): + pg_version = "%d.%d" % divmod(cr._obj.connection.server_version / 100, 100) + env = openerp.api.Environment(cr, SUPERUSER_ID, {}) + modules = dict([(i.name,i.latest_version) for i in env['ir.module.module'].search([('state','=','installed')])]) + manifest = { + 'odoo_dump': '1', + 'db_name': cr.dbname, + 'version': openerp.release.version, + 'version_info': openerp.release.version_info, + 'major_version': openerp.release.major_version, + 'pg_version': pg_version, + 'modules': modules, + } + return manifest + +def dump_db(db_name, stream, backup_format='zip'): """Dump database `db` into file-like object `stream`""" - with openerp.tools.osutil.tempdir() as dump_dir: - registry = openerp.modules.registry.RegistryManager.get(db) - with registry.cursor() as cr: - filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID) - if os.path.exists(filestore): - shutil.copytree(filestore, os.path.join(dump_dir, 'filestore')) - dump_file = os.path.join(dump_dir, 'dump.sql') - cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file] - if openerp.tools.config['db_user']: - cmd.append('--username=' + openerp.tools.config['db_user']) - if openerp.tools.config['db_host']: - cmd.append('--host=' + openerp.tools.config['db_host']) - if openerp.tools.config['db_port']: - cmd.append('--port=' + str(openerp.tools.config['db_port'])) - cmd.append(db) + cmd = ['pg_dump', '--no-owner'] + if openerp.tools.config['db_user']: + cmd.append('--username=' + openerp.tools.config['db_user']) + if openerp.tools.config['db_host']: + cmd.append('--host=' + openerp.tools.config['db_host']) + if openerp.tools.config['db_port']: + cmd.append('--port=' + str(openerp.tools.config['db_port'])) + cmd.append(db_name) - if openerp.tools.exec_pg_command(*cmd): - _logger.error('DUMP DB: %s failed! Please verify the configuration of the database ' - 'password on the server. You may need to create a .pgpass file for ' - 'authentication, or specify `db_password` in the server configuration ' - 'file.', db) - raise Exception("Couldn't dump database") + if backup_format == 'zip': + with openerp.tools.osutil.tempdir() as dump_dir: + registry = openerp.modules.registry.RegistryManager.get(db_name) + with registry.cursor() as cr: + filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID) + if os.path.exists(filestore): + shutil.copytree(filestore, os.path.join(dump_dir, 'filestore')) + manifest = dump_db_manifest(cr) + with open(os.path.join(dump_dir, 'manifest.json'), 'w') as fh: + json.dump(manifest, fh, indent=4) + cmd.insert(-1, '--file=' + os.path.join(dump_dir, 'dump.sql')) + openerp.tools.exec_pg_command(*cmd) + openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False) + else: + cmd.insert(-1, '--format=c') + print cmd + stdin, stdout = openerp.tools.exec_pg_command_pipe(*cmd) + shutil.copyfileobj(stdout, stream) - openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False) + _logger.info('DUMP DB successful: %s', db_name) - _logger.info('DUMP DB successful: %s', db) +def dump_db_stream(db_name, backup_format='zip'): + t=tempfile.TemporaryFile() + dump_db(db_name, t, backup_format) + t.seek(0) + return t def exp_restore(db_name, data, copy=False): data_file = tempfile.NamedTemporaryFile(delete=False) @@ -218,7 +216,6 @@ def exp_restore(db_name, data, copy=False): os.unlink(data_file.name) return True -@_set_pg_password_in_environment def restore_db(db, dump_file, copy=False): assert isinstance(db, basestring) if exp_db_exist(db): @@ -351,4 +348,3 @@ def exp_migrate_databases(databases): openerp.modules.registry.RegistryManager.new(db, force_demo=False, update_module=True) return True -# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index ca835f33b01..6dccab4b11e 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -65,6 +65,10 @@ _logger = logging.getLogger(__name__) # We include the *Base ones just in case, currently they seem to be subclasses of the _* ones. SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase) +#---------------------------------------------------------- +# Subprocesses +#---------------------------------------------------------- + def find_in_path(name): path = os.environ.get('PATH', os.defpath).split(os.pathsep) if config.get('bin_path') and config['bin_path'] != 'None': @@ -74,6 +78,24 @@ def find_in_path(name): except IOError: return None +def _exec_pipe(prog, args, env=None): + cmd = (prog,) + args + # on win32, passing close_fds=True is not compatible + # with redirecting std[in/err/out] + close_fds = os.name=="posix" + pop = subprocess.Popen(cmd, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=close_fds, env=env) + return pop.stdin, pop.stdout + +def exec_command_pipe(name, *args): + prog = find_in_path(name) + if not prog: + raise Exception('Command `%s` not found.' % name) + _exec_pipe(prog, *args) + +#---------------------------------------------------------- +# Postgres subprocesses +#---------------------------------------------------------- + def find_pg_tool(name): path = None if config['pg_path'] and config['pg_path'] != 'None': @@ -81,38 +103,33 @@ def find_pg_tool(name): try: return which(name, path=path) except IOError: - return None + raise Exception('Command `%s` not found.' % name) + +def exec_pg_environ(): + """ On systems where pg_restore/pg_dump require an explicit password (i.e. + on Windows where TCP sockets are used), it is necessary to pass the + postgres user password in the PGPASSWORD environment variable or in a + special .pgpass file. + + See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html + """ + env = os.environ.copy() + if not env.get('PGPASSWORD') and openerp.tools.config['db_password']: + env['PGPASSWORD'] = openerp.tools.config['db_password'] + return env def exec_pg_command(name, *args): prog = find_pg_tool(name) - if not prog: - raise Exception('Couldn\'t find %s' % name) - args2 = (prog,) + args - + env = exec_pg_environ() with open(os.devnull) as dn: - return subprocess.call(args2, stdout=dn, stderr=subprocess.STDOUT) + rc = subprocess.call((prog,) + args, stdout=dn, stderr=subprocess.STDOUT) + if rc: + raise Exception('Postgres subprocess %s error %s' % (args2, rc)) def exec_pg_command_pipe(name, *args): prog = find_pg_tool(name) - if not prog: - raise Exception('Couldn\'t find %s' % name) - # on win32, passing close_fds=True is not compatible - # with redirecting std[in/err/out] - pop = subprocess.Popen((prog,) + args, bufsize= -1, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - close_fds=(os.name=="posix")) - return pop.stdin, pop.stdout - -def exec_command_pipe(name, *args): - prog = find_in_path(name) - if not prog: - raise Exception('Couldn\'t find %s' % name) - # on win32, passing close_fds=True is not compatible - # with redirecting std[in/err/out] - pop = subprocess.Popen((prog,) + args, bufsize= -1, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - close_fds=(os.name=="posix")) - return pop.stdin, pop.stdout + env = exec_pg_environ() + return _exec_pipe(prog, args, env) #---------------------------------------------------------- # File paths