[FIX] dbmanager: backup support both zip and pg_dump custom format
- let the user choose between the pg_dump custom format or the zip format including the filestore - use file objects to allow dumps larger than memory - postgres subprocess invocation is now clean and thread-safe, we dont touch the local process environ anymore - add a manifest to the zip dump format with version information about odoo, postgres (pg_dump doesnt output it) and modules
This commit is contained in:
parent
f9376905cc
commit
ec9a543014
|
@ -724,21 +724,21 @@ class Database(http.Controller):
|
||||||
return {'error': _('Could not drop database !'), 'title': _('Drop Database')}
|
return {'error': _('Could not drop database !'), 'title': _('Drop Database')}
|
||||||
|
|
||||||
@http.route('/web/database/backup', type='http', auth="none")
|
@http.route('/web/database/backup', type='http', auth="none")
|
||||||
def backup(self, backup_db, backup_pwd, token):
|
def backup(self, backup_db, backup_pwd, token, backup_format='zip'):
|
||||||
try:
|
try:
|
||||||
db_dump = base64.b64decode(
|
openerp.service.security.check_super(backup_pwd)
|
||||||
request.session.proxy("db").dump(backup_pwd, backup_db))
|
ts = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S")
|
||||||
filename = "%(db)s_%(timestamp)s.dump" % {
|
filename = "%s_%s.%s" % (backup_db, ts, backup_format)
|
||||||
'db': backup_db,
|
headers = [
|
||||||
'timestamp': datetime.datetime.utcnow().strftime(
|
('Content-Type', 'application/octet-stream; charset=binary'),
|
||||||
"%Y-%m-%d_%H-%M-%SZ")
|
('Content-Disposition', content_disposition(filename)),
|
||||||
}
|
]
|
||||||
return request.make_response(db_dump,
|
dump_stream = openerp.service.db.dump_db_stream(backup_db, backup_format)
|
||||||
[('Content-Type', 'application/octet-stream; charset=binary'),
|
response = werkzeug.wrappers.Response(dump_stream, headers=headers, direct_passthrough=True)
|
||||||
('Content-Disposition', content_disposition(filename))],
|
response.set_cookie('fileToken', token)
|
||||||
{'fileToken': token}
|
return response
|
||||||
)
|
|
||||||
except Exception, e:
|
except Exception, e:
|
||||||
|
_logger.exception('Database.backup')
|
||||||
return simplejson.dumps([[],[{'error': openerp.tools.ustr(e), 'title': _('Backup Database')}]])
|
return simplejson.dumps([[],[{'error': openerp.tools.ustr(e), 'title': _('Backup Database')}]])
|
||||||
|
|
||||||
@http.route('/web/database/restore', type='http', auth="none")
|
@http.route('/web/database/restore', type='http', auth="none")
|
||||||
|
|
|
@ -225,6 +225,15 @@
|
||||||
<input t-if="!widget.db_list" name="backup_db" class="required" type="text" autofocus="autofocus"/>
|
<input t-if="!widget.db_list" name="backup_db" class="required" type="text" autofocus="autofocus"/>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><label for="backup_format">Backup format:</label></td>
|
||||||
|
<td class="oe_form_field oe_form_field_selection">
|
||||||
|
<select name="backup_format">
|
||||||
|
<option value="zip">zip (includes filestore)</option>
|
||||||
|
<option value="dump">pg_dump custom format (without filestore)</option>
|
||||||
|
</select>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><label for="backup_pwd">Master Password:</label></td>
|
<td><label for="backup_pwd">Master Password:</label></td>
|
||||||
<td><input type="password" name="backup_pwd" class="required" /></td>
|
<td><input type="password" name="backup_pwd" class="required" /></td>
|
||||||
|
|
|
@ -1,14 +1,17 @@
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
from contextlib import closing
|
|
||||||
from functools import wraps
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import traceback
|
import traceback
|
||||||
import tempfile
|
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
|
from functools import wraps
|
||||||
|
from contextlib import closing
|
||||||
|
|
||||||
import psycopg2
|
import psycopg2
|
||||||
|
|
||||||
import openerp
|
import openerp
|
||||||
|
@ -143,70 +146,65 @@ def exp_drop(db_name):
|
||||||
shutil.rmtree(fs)
|
shutil.rmtree(fs)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _set_pg_password_in_environment(func):
|
|
||||||
""" On systems where pg_restore/pg_dump require an explicit
|
|
||||||
password (i.e. when not connecting via unix sockets, and most
|
|
||||||
importantly on Windows), it is necessary to pass the PG user
|
|
||||||
password in the environment or in a special .pgpass file.
|
|
||||||
|
|
||||||
This decorator handles setting
|
|
||||||
:envvar:`PGPASSWORD` if it is not already
|
|
||||||
set, and removing it afterwards.
|
|
||||||
|
|
||||||
See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
|
|
||||||
|
|
||||||
.. note:: This is not thread-safe, and should never be enabled for
|
|
||||||
SaaS (giving SaaS users the super-admin password is not a good idea
|
|
||||||
anyway)
|
|
||||||
"""
|
|
||||||
@wraps(func)
|
|
||||||
def wrapper(*args, **kwargs):
|
|
||||||
if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
|
|
||||||
return func(*args, **kwargs)
|
|
||||||
else:
|
|
||||||
os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
|
|
||||||
try:
|
|
||||||
return func(*args, **kwargs)
|
|
||||||
finally:
|
|
||||||
del os.environ['PGPASSWORD']
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
def exp_dump(db_name):
|
def exp_dump(db_name):
|
||||||
with tempfile.TemporaryFile() as t:
|
with tempfile.TemporaryFile() as t:
|
||||||
dump_db(db_name, t)
|
dump_db(db_name, t)
|
||||||
t.seek(0)
|
t.seek(0)
|
||||||
return t.read().encode('base64')
|
return t.read().encode('base64')
|
||||||
|
|
||||||
@_set_pg_password_in_environment
|
def dump_db_manifest(cr):
|
||||||
def dump_db(db, stream):
|
pg_version = "%d.%d" % divmod(cr._obj.connection.server_version / 100, 100)
|
||||||
|
env = openerp.api.Environment(cr, SUPERUSER_ID, {})
|
||||||
|
modules = dict([(i.name,i.latest_version) for i in env['ir.module.module'].search([('state','=','installed')])])
|
||||||
|
manifest = {
|
||||||
|
'odoo_dump': '1',
|
||||||
|
'db_name': cr.dbname,
|
||||||
|
'version': openerp.release.version,
|
||||||
|
'version_info': openerp.release.version_info,
|
||||||
|
'major_version': openerp.release.major_version,
|
||||||
|
'pg_version': pg_version,
|
||||||
|
'modules': modules,
|
||||||
|
}
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
def dump_db(db_name, stream, backup_format='zip'):
|
||||||
"""Dump database `db` into file-like object `stream`"""
|
"""Dump database `db` into file-like object `stream`"""
|
||||||
with openerp.tools.osutil.tempdir() as dump_dir:
|
|
||||||
registry = openerp.modules.registry.RegistryManager.get(db)
|
|
||||||
with registry.cursor() as cr:
|
|
||||||
filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
|
|
||||||
if os.path.exists(filestore):
|
|
||||||
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
|
|
||||||
|
|
||||||
dump_file = os.path.join(dump_dir, 'dump.sql')
|
cmd = ['pg_dump', '--no-owner']
|
||||||
cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
|
if openerp.tools.config['db_user']:
|
||||||
if openerp.tools.config['db_user']:
|
cmd.append('--username=' + openerp.tools.config['db_user'])
|
||||||
cmd.append('--username=' + openerp.tools.config['db_user'])
|
if openerp.tools.config['db_host']:
|
||||||
if openerp.tools.config['db_host']:
|
cmd.append('--host=' + openerp.tools.config['db_host'])
|
||||||
cmd.append('--host=' + openerp.tools.config['db_host'])
|
if openerp.tools.config['db_port']:
|
||||||
if openerp.tools.config['db_port']:
|
cmd.append('--port=' + str(openerp.tools.config['db_port']))
|
||||||
cmd.append('--port=' + str(openerp.tools.config['db_port']))
|
cmd.append(db_name)
|
||||||
cmd.append(db)
|
|
||||||
|
|
||||||
if openerp.tools.exec_pg_command(*cmd):
|
if backup_format == 'zip':
|
||||||
_logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
|
with openerp.tools.osutil.tempdir() as dump_dir:
|
||||||
'password on the server. You may need to create a .pgpass file for '
|
registry = openerp.modules.registry.RegistryManager.get(db_name)
|
||||||
'authentication, or specify `db_password` in the server configuration '
|
with registry.cursor() as cr:
|
||||||
'file.', db)
|
filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
|
||||||
raise Exception("Couldn't dump database")
|
if os.path.exists(filestore):
|
||||||
|
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
|
||||||
|
manifest = dump_db_manifest(cr)
|
||||||
|
with open(os.path.join(dump_dir, 'manifest.json'), 'w') as fh:
|
||||||
|
json.dump(manifest, fh, indent=4)
|
||||||
|
cmd.insert(-1, '--file=' + os.path.join(dump_dir, 'dump.sql'))
|
||||||
|
openerp.tools.exec_pg_command(*cmd)
|
||||||
|
openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
|
||||||
|
else:
|
||||||
|
cmd.insert(-1, '--format=c')
|
||||||
|
print cmd
|
||||||
|
stdin, stdout = openerp.tools.exec_pg_command_pipe(*cmd)
|
||||||
|
shutil.copyfileobj(stdout, stream)
|
||||||
|
|
||||||
openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
|
_logger.info('DUMP DB successful: %s', db_name)
|
||||||
|
|
||||||
_logger.info('DUMP DB successful: %s', db)
|
def dump_db_stream(db_name, backup_format='zip'):
|
||||||
|
t=tempfile.TemporaryFile()
|
||||||
|
dump_db(db_name, t, backup_format)
|
||||||
|
t.seek(0)
|
||||||
|
return t
|
||||||
|
|
||||||
def exp_restore(db_name, data, copy=False):
|
def exp_restore(db_name, data, copy=False):
|
||||||
data_file = tempfile.NamedTemporaryFile(delete=False)
|
data_file = tempfile.NamedTemporaryFile(delete=False)
|
||||||
|
@ -218,7 +216,6 @@ def exp_restore(db_name, data, copy=False):
|
||||||
os.unlink(data_file.name)
|
os.unlink(data_file.name)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@_set_pg_password_in_environment
|
|
||||||
def restore_db(db, dump_file, copy=False):
|
def restore_db(db, dump_file, copy=False):
|
||||||
assert isinstance(db, basestring)
|
assert isinstance(db, basestring)
|
||||||
if exp_db_exist(db):
|
if exp_db_exist(db):
|
||||||
|
@ -351,4 +348,3 @@ def exp_migrate_databases(databases):
|
||||||
openerp.modules.registry.RegistryManager.new(db, force_demo=False, update_module=True)
|
openerp.modules.registry.RegistryManager.new(db, force_demo=False, update_module=True)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
|
|
||||||
|
|
|
@ -65,6 +65,10 @@ _logger = logging.getLogger(__name__)
|
||||||
# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
|
# We include the *Base ones just in case, currently they seem to be subclasses of the _* ones.
|
||||||
SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase)
|
SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.CommentBase, etree.PIBase)
|
||||||
|
|
||||||
|
#----------------------------------------------------------
|
||||||
|
# Subprocesses
|
||||||
|
#----------------------------------------------------------
|
||||||
|
|
||||||
def find_in_path(name):
|
def find_in_path(name):
|
||||||
path = os.environ.get('PATH', os.defpath).split(os.pathsep)
|
path = os.environ.get('PATH', os.defpath).split(os.pathsep)
|
||||||
if config.get('bin_path') and config['bin_path'] != 'None':
|
if config.get('bin_path') and config['bin_path'] != 'None':
|
||||||
|
@ -74,6 +78,24 @@ def find_in_path(name):
|
||||||
except IOError:
|
except IOError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def _exec_pipe(prog, args, env=None):
|
||||||
|
cmd = (prog,) + args
|
||||||
|
# on win32, passing close_fds=True is not compatible
|
||||||
|
# with redirecting std[in/err/out]
|
||||||
|
close_fds = os.name=="posix"
|
||||||
|
pop = subprocess.Popen(cmd, bufsize=-1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=close_fds, env=env)
|
||||||
|
return pop.stdin, pop.stdout
|
||||||
|
|
||||||
|
def exec_command_pipe(name, *args):
|
||||||
|
prog = find_in_path(name)
|
||||||
|
if not prog:
|
||||||
|
raise Exception('Command `%s` not found.' % name)
|
||||||
|
_exec_pipe(prog, *args)
|
||||||
|
|
||||||
|
#----------------------------------------------------------
|
||||||
|
# Postgres subprocesses
|
||||||
|
#----------------------------------------------------------
|
||||||
|
|
||||||
def find_pg_tool(name):
|
def find_pg_tool(name):
|
||||||
path = None
|
path = None
|
||||||
if config['pg_path'] and config['pg_path'] != 'None':
|
if config['pg_path'] and config['pg_path'] != 'None':
|
||||||
|
@ -81,38 +103,33 @@ def find_pg_tool(name):
|
||||||
try:
|
try:
|
||||||
return which(name, path=path)
|
return which(name, path=path)
|
||||||
except IOError:
|
except IOError:
|
||||||
return None
|
raise Exception('Command `%s` not found.' % name)
|
||||||
|
|
||||||
|
def exec_pg_environ():
|
||||||
|
""" On systems where pg_restore/pg_dump require an explicit password (i.e.
|
||||||
|
on Windows where TCP sockets are used), it is necessary to pass the
|
||||||
|
postgres user password in the PGPASSWORD environment variable or in a
|
||||||
|
special .pgpass file.
|
||||||
|
|
||||||
|
See also http://www.postgresql.org/docs/8.4/static/libpq-envars.html
|
||||||
|
"""
|
||||||
|
env = os.environ.copy()
|
||||||
|
if not env.get('PGPASSWORD') and openerp.tools.config['db_password']:
|
||||||
|
env['PGPASSWORD'] = openerp.tools.config['db_password']
|
||||||
|
return env
|
||||||
|
|
||||||
def exec_pg_command(name, *args):
|
def exec_pg_command(name, *args):
|
||||||
prog = find_pg_tool(name)
|
prog = find_pg_tool(name)
|
||||||
if not prog:
|
env = exec_pg_environ()
|
||||||
raise Exception('Couldn\'t find %s' % name)
|
|
||||||
args2 = (prog,) + args
|
|
||||||
|
|
||||||
with open(os.devnull) as dn:
|
with open(os.devnull) as dn:
|
||||||
return subprocess.call(args2, stdout=dn, stderr=subprocess.STDOUT)
|
rc = subprocess.call((prog,) + args, stdout=dn, stderr=subprocess.STDOUT)
|
||||||
|
if rc:
|
||||||
|
raise Exception('Postgres subprocess %s error %s' % (args2, rc))
|
||||||
|
|
||||||
def exec_pg_command_pipe(name, *args):
|
def exec_pg_command_pipe(name, *args):
|
||||||
prog = find_pg_tool(name)
|
prog = find_pg_tool(name)
|
||||||
if not prog:
|
env = exec_pg_environ()
|
||||||
raise Exception('Couldn\'t find %s' % name)
|
return _exec_pipe(prog, args, env)
|
||||||
# on win32, passing close_fds=True is not compatible
|
|
||||||
# with redirecting std[in/err/out]
|
|
||||||
pop = subprocess.Popen((prog,) + args, bufsize= -1,
|
|
||||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
||||||
close_fds=(os.name=="posix"))
|
|
||||||
return pop.stdin, pop.stdout
|
|
||||||
|
|
||||||
def exec_command_pipe(name, *args):
|
|
||||||
prog = find_in_path(name)
|
|
||||||
if not prog:
|
|
||||||
raise Exception('Couldn\'t find %s' % name)
|
|
||||||
# on win32, passing close_fds=True is not compatible
|
|
||||||
# with redirecting std[in/err/out]
|
|
||||||
pop = subprocess.Popen((prog,) + args, bufsize= -1,
|
|
||||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
||||||
close_fds=(os.name=="posix"))
|
|
||||||
return pop.stdin, pop.stdout
|
|
||||||
|
|
||||||
#----------------------------------------------------------
|
#----------------------------------------------------------
|
||||||
# File paths
|
# File paths
|
||||||
|
|
Loading…
Reference in New Issue