odoo
/
odoo
2
0
Fork 0
Code Pull Requests Releases Activity

[IMP] new backup format. 

The new backup format is a zip containing the sql dump of the
database and the filestore of this database.

Old backups can still be restored

bzr revid: chs@openerp.com-20140304175656-iu3un6q43ttnhjfz
This commit is contained in:
Christophe Simonis 2014-03-04 18:56:56 +01:00
parent ab80456fe7
commit 447d597815
2 changed files with 135 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
openerp/service/db.py
View File

@ -1,12 +1,15 @@
# -*- coding: utf-8 -*-
import base64
import contextlib
from contextlib import closing
from functools import wraps
import logging
import os
import shutil
import threading
import traceback
from contextlib import closing
import tempfile
import zipfile
import psycopg2
import openerp
from openerp import SUPERUSER_ID
@ -141,7 +144,8 @@ def exp_get_progress(id):
return 1.0, users
else:
a = self_actions.pop(id)
raise Exception, a['exception'], a['traceback'] # flake8: noqa
exc, tb = a['exception'], a['traceback']
raise Exception, exc, tb
def exp_drop(db_name):
if db_name not in exp_list(True):
@ -176,14 +180,13 @@ def exp_drop(db_name):
_logger.info('DROP DB: %s', db_name)
return True
@contextlib.contextmanager
def _set_pg_password_in_environment():
def _set_pg_password_in_environment(func):
""" On systems where pg_restore/pg_dump require an explicit
password (i.e. when not connecting via unix sockets, and most
importantly on Windows), it is necessary to pass the PG user
password in the environment or in a special .pgpass file.
This context management method handles setting
This decorator handles setting
:envvar:`PGPASSWORD` if it is not already
set, and removing it afterwards.
@ -193,77 +196,124 @@ def _set_pg_password_in_environment():
SaaS (giving SaaS users the super-admin password is not a good idea
anyway)
"""
if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
yield
else:
os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
try:
yield
finally:
del os.environ['PGPASSWORD']
@wraps(func)
def wrapper(*args, **kwargs):
if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
return func(*args, **kwargs)
else:
os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
try:
return func(*args, **kwargs)
finally:
del os.environ['PGPASSWORD']
return wrapper
def exp_dump(db_name):
with _set_pg_password_in_environment():
cmd = ['pg_dump', '--format=c', '--no-owner']
with tempfile.TemporaryFile() as t:
dump_db(db_name, t)
t.seek(0)
return t.read().encode('base64')
@_set_pg_password_in_environment
def dump_db(db, stream):
"""Dump database `db` into file-like object `stream`"""
with openerp.tools.osutil.tempdir() as dump_dir:
registry = openerp.modules.registry.RegistryManager.get(db)
with registry.cursor() as cr:
filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
if os.path.exists(filestore):
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
dump_file = os.path.join(dump_dir, 'dump.sql')
cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
if openerp.tools.config['db_user']:
cmd.append('--username=' + openerp.tools.config['db_user'])
if openerp.tools.config['db_host']:
cmd.append('--host=' + openerp.tools.config['db_host'])
if openerp.tools.config['db_port']:
cmd.append('--port=' + str(openerp.tools.config['db_port']))
cmd.append(db_name)
cmd.append(db)
stdin, stdout = openerp.tools.exec_pg_command_pipe(*tuple(cmd))
stdin.close()
data = stdout.read()
res = stdout.close()
if not data or res:
if openerp.tools.exec_pg_command(*cmd):
_logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
'password on the server. You may need to create a .pgpass file for '
'authentication, or specify `db_password` in the server configuration '
'file.\n %s', db_name, data)
'file.', db)
raise Exception("Couldn't dump database")
_logger.info('DUMP DB successful: %s', db_name)
return base64.encodestring(data)
openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
def exp_restore(db_name, data):
with _set_pg_password_in_environment():
if exp_db_exist(db_name):
_logger.warning('RESTORE DB: %s already exists', db_name)
raise Exception("Database already exists")
_logger.info('DUMP DB successful: %s', db)
_create_empty_database(db_name)
def exp_restore(db_name, data, copy=False):
data_file = tempfile.NamedTemporaryFile(delete=False)
try:
data_file.write(data.decode('base64'))
data_file.close()
restore_db(db_name, data_file.name, copy=copy)
finally:
os.unlink(data_file.name)
return True
cmd = ['pg_restore', '--no-owner']
@_set_pg_password_in_environment
def restore_db(db, dump_file, copy=False):
assert isinstance(db, basestring)
if exp_db_exist(db):
_logger.warning('RESTORE DB: %s already exists', db)
raise Exception("Database already exists")
_create_empty_database(db)
filestore_path = None
with openerp.tools.osutil.tempdir() as dump_dir:
if zipfile.is_zipfile(dump_file):
# v8 format
with zipfile.ZipFile(dump_file, 'r') as z:
# only extract known members!
filestore = [m for m in z.namelist() if m.startswith('filestore/')]
z.extractall(dump_dir, ['dump.sql'] + filestore)
if filestore:
filestore_path = os.path.join(dump_dir, 'filestore')
pg_cmd = 'psql'
pg_args = ['-q', '-f', os.path.join(dump_dir, 'dump.sql')]
else:
# <= 7.0 format (raw pg_dump output)
pg_cmd = 'pg_restore'
pg_args = ['--no-owner', dump_file]
args = []
if openerp.tools.config['db_user']:
cmd.append('--username=' + openerp.tools.config['db_user'])
args.append('--username=' + openerp.tools.config['db_user'])
if openerp.tools.config['db_host']:
cmd.append('--host=' + openerp.tools.config['db_host'])
args.append('--host=' + openerp.tools.config['db_host'])
if openerp.tools.config['db_port']:
cmd.append('--port=' + str(openerp.tools.config['db_port']))
cmd.append('--dbname=' + db_name)
args2 = tuple(cmd)
args.append('--port=' + str(openerp.tools.config['db_port']))
args.append('--dbname=' + db)
pg_args = args + pg_args
buf = base64.decodestring(data)
if os.name == "nt":
tmpfile = (os.environ['TMP'] or 'C:\\') + os.tmpnam()
file(tmpfile, 'wb').write(buf)
args2 = list(args2)
args2.append(tmpfile)
args2 = tuple(args2)
stdin, stdout = openerp.tools.exec_pg_command_pipe(*args2)
if not os.name == "nt":
stdin.write(base64.decodestring(data))
stdin.close()
res = stdout.close()
if res:
if openerp.tools.exec_pg_command(pg_cmd, *pg_args):
raise Exception("Couldn't restore database")
_logger.info('RESTORE DB: %s', db_name)
return True
registry = openerp.modules.registry.RegistryManager.new(db)
with registry.cursor() as cr:
if copy:
# if it's a copy of a database, force generation of a new dbuuid
registry['ir.config_parameter'].init(cr, force=True)
if filestore_path:
filestore_dest = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
shutil.move(filestore_path, filestore_dest)
if openerp.tools.config['unaccent']:
try:
with cr.savepoint():
cr.execute("CREATE EXTENSION unaccent")
except psycopg2.Error:
pass
_logger.info('RESTORE DB: %s', db)
def exp_rename(old_name, new_name):
openerp.modules.registry.RegistryManager.delete(old_name)
@ -280,6 +330,7 @@ def exp_rename(old_name, new_name):
raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e))
return True
@openerp.tools.mute_logger('openerp.sql_db')
def exp_db_exist(db_name):
## Not True: in fact, check if connection to database is possible. The database may exists
return bool(openerp.sql_db.db_connect(db_name))

28
openerp/tools/osutil.py
View File

@ -23,8 +23,12 @@
Some functions related to the os and os.path module
"""
from contextlib import contextmanager
import os
from os.path import join as opj
import shutil
import tempfile
import zipfile
if os.name == 'nt':
import ctypes
@ -61,6 +65,30 @@ def walksymlinks(top, topdown=True, onerror=None):
if not topdown:
yield dirpath, dirnames, filenames
@contextmanager
def tempdir():
tmpdir = tempfile.mkdtemp()
try:
yield tmpdir
finally:
shutil.rmtree(tmpdir)
def zip_dir(path, stream, include_dir=True): # TODO add ignore list
path = os.path.normpath(path)
len_prefix = len(os.path.dirname(path)) if include_dir else len(path)
if len_prefix:
len_prefix += 1
with zipfile.ZipFile(stream, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zipf:
for dirpath, dirnames, filenames in os.walk(path):
for fname in filenames:
bname, ext = os.path.splitext(fname)
ext = ext or bname
if ext not in ['.pyc', '.pyo', '.swp', '.DS_Store']:
path = os.path.normpath(os.path.join(dirpath, fname))
if os.path.isfile(path):
zipf.write(path, path[len_prefix:])
if os.name != 'nt':
getppid = os.getppid