[FIX] dbmanager: backup support both zip and pg_dump custom format
- let the user choose between the pg_dump custom format or the zip format including the filestore - use file objects to allow dumps larger than memory - postgres subprocess invocation is now clean and thread-safe, we dont touch the local process environ anymore - add a manifest to the zip dump format with version information about odoo, postgres (pg_dump doesnt output it) and modules
This commit is contained in:
Antony Lesuisse
parent
f9376905cc
commit
ec9a543014
|
|
@ -724,21 +724,21 @@ class Database(http.Controller):
|
|||
return {'error': _('Could not drop database !'), 'title': _('Drop Database')}
|
||||
|
||||
@http.route('/web/database/backup', type='http', auth="none")
|
||||
def backup(self, backup_db, backup_pwd, token):
|
||||
def backup(self, backup_db, backup_pwd, token, backup_format='zip'):
|
||||
try:
|
||||
db_dump = base64.b64decode(
|
||||
request.session.proxy("db").dump(backup_pwd, backup_db))
|
||||
filename = "%(db)s_%(timestamp)s.dump" % {
|
||||
'db': backup_db,
|
||||
'timestamp': datetime.datetime.utcnow().strftime(
|
||||
"%Y-%m-%d_%H-%M-%SZ")
|
||||
}
|
||||
return request.make_response(db_dump,
|
||||
[('Content-Type', 'application/octet-stream; charset=binary'),
|
||||
('Content-Disposition', content_disposition(filename))],
|
||||
{'fileToken': token}
|
||||
)
|
||||
openerp.service.security.check_super(backup_pwd)
|
||||
ts = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
filename = "%s_%s.%s" % (backup_db, ts, backup_format)
|
||||
headers = [
|
||||
('Content-Type', 'application/octet-stream; charset=binary'),
|
||||
('Content-Disposition', content_disposition(filename)),
|
||||
]
|
||||
dump_stream = openerp.service.db.dump_db_stream(backup_db, backup_format)
|
||||
response = werkzeug.wrappers.Response(dump_stream, headers=headers, direct_passthrough=True)
|
||||
response.set_cookie('fileToken', token)
|
||||
return response
|
||||
except Exception, e:
|
||||
_logger.exception('Database.backup')
|
||||
return simplejson.dumps([[],[{'error': openerp.tools.ustr(e), 'title': _('Backup Database')}]])
|
||||
|
||||
@http.route('/web/database/restore', type='http', auth="none")
|
||||
|
|
|
|||
|
|
@ -225,6 +225,15 @@
|
|||
<input t-if="!widget.db_list" name="backup_db" class="required" type="text" autofocus="autofocus"/>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><label for="backup_format">Backup format:</label></td>
|
||||
<td class="oe_form_field oe_form_field_selection">
|
||||
<select name="backup_format">
|
||||
<option value="zip">zip (includes filestore)</option>
|
||||
<option value="dump">pg_dump custom format (without filestore)</option>
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><label for="backup_pwd">Master Password:</label></td>
|
||||
<td><input type="password" name="backup_pwd" class="required" /></td>
|
||||
|
|
|
|||
|
|
@ -1,14 +1,17 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from contextlib import closing
|
||||
from functools import wraps
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
import traceback
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
from functools import wraps
|
||||
from contextlib import closing
|
||||
|
||||
import psycopg2
|
||||
|
||||
import openerp
|
||||
|
|
@ -143,70 +146,65 @@ def exp_drop(db_name):
|
|||
shutil.rmtree(fs)
|
||||
return True
|
||||
|
||||
def _set_pg_password_in_environment(func):
|
||||
""" On systems where pg_restore/pg_dump require an explicit
|
||||
password (i.e. when not connecting via unix sockets, and most
|
||||
importantly on Windows), it is necessary to pass the PG user
|
||||
password in the environment or in a special .pgpass file.
|
||||
|
||||
This decorator handles setting
|
||||
:envvar:`PGPASSWORD` if it is not already
|
||||
set, and removing it afterwards.
|
||||
|
||||
See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
|
||||
|
||||
.. note:: This is not thread-safe, and should never be enabled for
|
||||
SaaS (giving SaaS users the super-admin password is not a good idea
|
||||
anyway)
|
||||
"""
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
|
||||
return func(*args, **kwargs)
|
||||
else:
|
||||
os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
del os.environ['PGPASSWORD']
|
||||
return wrapper
|
||||
|
||||
def exp_dump(db_name):
|
||||
with tempfile.TemporaryFile() as t:
|
||||
dump_db(db_name, t)
|
||||
t.seek(0)
|
||||
return t.read().encode('base64')
|
||||
|
||||
@_set_pg_password_in_environment
|
||||
def dump_db(db, stream):
|
||||
def dump_db_manifest(cr):
|
||||
pg_version = "%d.%d" % divmod(cr._obj.connection.server_version / 100, 100)
|
||||
env = openerp.api.Environment(cr, SUPERUSER_ID, {})
|
||||
modules = dict([(i.name,i.latest_version) for i in env['ir.module.module'].search([('state','=','installed')])])
|
||||
manifest = {
|
||||
'odoo_dump': '1',
|
||||
'db_name': cr.dbname,
|
||||
'version': openerp.release.version,
|
||||
'version_info': openerp.release.version_info,
|
||||
'major_version': openerp.release.major_version,
|
||||
'pg_version': pg_version,
|
||||
'modules': modules,
|
||||
}
|
||||
return manifest
|
||||
|
||||
def dump_db(db_name, stream, backup_format='zip'):
|
||||
"""Dump database `db` into file-like object `stream`"""
|
||||
with openerp.tools.osutil.tempdir() as dump_dir:
|
||||
registry = openerp.modules.registry.RegistryManager.get(db)
|
||||
with registry.cursor() as cr:
|
||||
filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
|
||||
if os.path.exists(filestore):
|
||||
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
|
||||
|
||||
dump_file = os.path.join(dump_dir, 'dump.sql')
|
||||
cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
|
||||
if openerp.tools.config['db_user']:
|
||||
cmd.append('--username=' + openerp.tools.config['db_user'])
|
||||
if openerp.tools.config['db_host']:
|
||||
cmd.append('--host=' + openerp.tools.config['db_host'])
|
||||
if openerp.tools.config['db_port']:
|
||||
cmd.append('--port=' + str(openerp.tools.config['db_port']))
|
||||
cmd.append(db)
|
||||
cmd = ['pg_dump', '--no-owner']
|
||||
if openerp.tools.config['db_user']:
|
||||
cmd.append('--username=' + openerp.tools.config['db_user'])
|
||||
if openerp.tools.config['db_host']:
|
||||
cmd.append('--host=' + openerp.tools.config['db_host'])
|
||||
if openerp.tools.config['db_port']:
|
||||
cmd.append('--port=' + str(openerp.tools.config['db_port']))
|
||||
cmd.append(db_name)
|
||||
|
||||
if openerp.tools.exec_pg_command(*cmd):
|
||||
_logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
|
||||
'password on the server. You may need to create a .pgpass file for '
|
||||
'authentication, or specify `db_password` in the server configuration '
|
||||
'file.', db)
|
||||
raise Exception("Couldn't dump database")
|
||||
if backup_format == 'zip':
|
||||
with openerp.tools.osutil.tempdir() as dump_dir:
|
||||
registry = openerp.modules.registry.RegistryManager.get(db_name)
|
||||
with registry.cursor() as cr:
|
||||
filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
|
||||
if os.path.exists(filestore):
|
||||
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
|
||||
manifest = dump_db_manifest(cr)
|
||||
with open(os.path.join(dump_dir, 'manifest.json'), 'w') as fh:
|
||||
json.dump(manifest, fh, indent=4)
|
||||
cmd.insert(-1, '--file=' + os.path.join(dump_dir, 'dump.sql'))
|
||||
openerp.tools.exec_pg_command(*cmd)
|
||||
openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
|
||||
else:
|
||||
cmd.insert(-1, '--format=c')
|
||||
print cmd
|
||||
stdin, stdout = openerp.tools.exec_pg_command_pipe(*cmd)
|
||||
shutil.copyfileobj(stdout, stream)
|
||||
|
||||
openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
|
||||
_logger.info('DUMP DB successful: %s', db_name)
|
||||
|
||||
_logger.info('DUMP DB successful: %s', db)
|
||||
def dump_db_stream(db_name, backup_format='zip'):
|
||||
t=tempfile.TemporaryFile()
|
||||
dump_db(db_name, t, backup_format)
|
||||
t.seek(0)
|
||||
return t
|
||||
|
||||
def exp_restore(db_name, data, copy=False):
|
||||
data_file = tempfile.NamedTemporaryFile(delete=False)
|
||||
|
|
@ -218,7 +216,6 @@ def exp_restore(db_name, data, copy=False):
|
|||
os.unlink(data_file.name)
|
||||
return True
|
||||
|
||||
@_set_pg_password_in_environment
|
||||
def restore_db(db, dump_file, copy=False):
|
||||
assert isinstance(db, basestring)
|
||||
if exp_db_exist(db):
|
||||
|
|
@ -351,4 +348,3 @@ def exp_migrate_databases(databases):
|
|||
openerp.modules.registry.RegistryManager.new(db, force_demo=False, update_module=True)
|
||||
return True
|
||||
|
||||
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
|
||||
|
|
|
|||
|
|
@ -65,6 +65,10 @@ _logger = logging.getLogger(__name__)
|
|||
# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
|
||||
SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase)
|
||||
|
||||
#----------------------------------------------------------
|
||||
# Subprocesses
|
||||
#----------------------------------------------------------
|
||||
|
||||
def find_in_path(name):
|
||||
path = os.environ.get('PATH', os.defpath).split(os.pathsep)
|
||||
if config.get('bin_path') and config['bin_path'] != 'None':
|
||||
|
|
@ -74,6 +78,24 @@ def find_in_path(name):
|
|||
except IOError:
|
||||
return None
|
||||
|
||||
def _exec_pipe(prog, args, env=None):
|
||||
cmd = (prog,) + args
|
||||
# on win32, passing close_fds=True is not compatible
|
||||
# with redirecting std[in/err/out]
|
||||
close_fds = os.name=="posix"
|
||||
pop = subprocess.Popen(cmd, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=close_fds, env=env)
|
||||
return pop.stdin, pop.stdout
|
||||
|
||||
def exec_command_pipe(name, *args):
|
||||
prog = find_in_path(name)
|
||||
if not prog:
|
||||
raise Exception('Command `%s` not found.' % name)
|
||||
_exec_pipe(prog, *args)
|
||||
|
||||
#----------------------------------------------------------
|
||||
# Postgres subprocesses
|
||||
#----------------------------------------------------------
|
||||
|
||||
def find_pg_tool(name):
|
||||
path = None
|
||||
if config['pg_path'] and config['pg_path'] != 'None':
|
||||
|
|
@ -81,38 +103,33 @@ def find_pg_tool(name):
|
|||
try:
|
||||
return which(name, path=path)
|
||||
except IOError:
|
||||
return None
|
||||
raise Exception('Command `%s` not found.' % name)
|
||||
|
||||
def exec_pg_environ():
|
||||
""" On systems where pg_restore/pg_dump require an explicit password (i.e.
|
||||
on Windows where TCP sockets are used), it is necessary to pass the
|
||||
postgres user password in the PGPASSWORD environment variable or in a
|
||||
special .pgpass file.
|
||||
|
||||
See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
if not env.get('PGPASSWORD') and openerp.tools.config['db_password']:
|
||||
env['PGPASSWORD'] = openerp.tools.config['db_password']
|
||||
return env
|
||||
|
||||
def exec_pg_command(name, *args):
|
||||
prog = find_pg_tool(name)
|
||||
if not prog:
|
||||
raise Exception('Couldn\'t find %s' % name)
|
||||
args2 = (prog,) + args
|
||||
|
||||
env = exec_pg_environ()
|
||||
with open(os.devnull) as dn:
|
||||
return subprocess.call(args2, stdout=dn, stderr=subprocess.STDOUT)
|
||||
rc = subprocess.call((prog,) + args, stdout=dn, stderr=subprocess.STDOUT)
|
||||
if rc:
|
||||
raise Exception('Postgres subprocess %s error %s' % (args2, rc))
|
||||
|
||||
def exec_pg_command_pipe(name, *args):
|
||||
prog = find_pg_tool(name)
|
||||
if not prog:
|
||||
raise Exception('Couldn\'t find %s' % name)
|
||||
# on win32, passing close_fds=True is not compatible
|
||||
# with redirecting std[in/err/out]
|
||||
pop = subprocess.Popen((prog,) + args, bufsize= -1,
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
close_fds=(os.name=="posix"))
|
||||
return pop.stdin, pop.stdout
|
||||
|
||||
def exec_command_pipe(name, *args):
|
||||
prog = find_in_path(name)
|
||||
if not prog:
|
||||
raise Exception('Couldn\'t find %s' % name)
|
||||
# on win32, passing close_fds=True is not compatible
|
||||
# with redirecting std[in/err/out]
|
||||
pop = subprocess.Popen((prog,) + args, bufsize= -1,
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
close_fds=(os.name=="posix"))
|
||||
return pop.stdin, pop.stdout
|
||||
env = exec_pg_environ()
|
||||
return _exec_pipe(prog, args, env)
|
||||
|
||||
#----------------------------------------------------------
|
||||
# File paths
|
||||
|
|
|
|||
Loading…
Reference in New Issue