[IMP] new backup format.

The new backup format is a zip containing the sql dump of the
database and the filestore of this database.

Old backups can still be restored

bzr revid: chs@openerp.com-20140304175656-iu3un6q43ttnhjfz

openerp/service/db.py

@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
-
+from contextlib import closing
-import base64
+from functools import wraps
-import contextlib
 import logging
 import os
+import shutil
 import threading
 import traceback
-from contextlib import closing
+import tempfile
+import zipfile
+
+import psycopg2
 
 import openerp
 from openerp import SUPERUSER_ID
@@ -141,7 +144,8 @@ def exp_get_progress(id):
             return 1.0, users
         else:
             a = self_actions.pop(id)
-            raise Exception, a['exception'], a['traceback']     # flake8: noqa
+            exc, tb = a['exception'], a['traceback']
+            raise Exception, exc, tb
 
 def exp_drop(db_name):
     if db_name not in exp_list(True):
@@ -176,14 +180,13 @@ def exp_drop(db_name):
             _logger.info('DROP DB: %s', db_name)
     return True
 
-@contextlib.contextmanager
+def _set_pg_password_in_environment(func):
-def _set_pg_password_in_environment():
     """ On systems where pg_restore/pg_dump require an explicit
     password (i.e. when not connecting via unix sockets, and most
     importantly on Windows), it is necessary to pass the PG user
     password in the environment or in a special .pgpass file.
 
-    This context management method handles setting
+    This decorator handles setting
     :envvar:`PGPASSWORD` if it is not already
     set, and removing it afterwards.
 
@@ -193,77 +196,124 @@ def _set_pg_password_in_environment():
          SaaS (giving SaaS users the super-admin password is not a good idea
          anyway)
     """
-    if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
+    @wraps(func)
-        yield
+    def wrapper(*args, **kwargs):
-    else:
+        if os.environ.get('PGPASSWORD') or not openerp.tools.config['db_password']:
-        os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
+            return func(*args, **kwargs)
-        try:
+        else:
-            yield
+            os.environ['PGPASSWORD'] = openerp.tools.config['db_password']
-        finally:
+            try:
-            del os.environ['PGPASSWORD']
+                return func(*args, **kwargs)
-
+            finally:
+                del os.environ['PGPASSWORD']
+    return wrapper
 
 def exp_dump(db_name):
-    with _set_pg_password_in_environment():
+    with tempfile.TemporaryFile() as t:
-        cmd = ['pg_dump', '--format=c', '--no-owner']
+        dump_db(db_name, t)
+        t.seek(0)
+        return t.read().encode('base64')
+
+@_set_pg_password_in_environment
+def dump_db(db, stream):
+    """Dump database `db` into file-like object `stream`"""
+    with openerp.tools.osutil.tempdir() as dump_dir:
+        registry = openerp.modules.registry.RegistryManager.get(db)
+        with registry.cursor() as cr:
+            filestore = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
+            if os.path.exists(filestore):
+                shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
+
+        dump_file = os.path.join(dump_dir, 'dump.sql')
+        cmd = ['pg_dump', '--format=p', '--no-owner', '--file=' + dump_file]
         if openerp.tools.config['db_user']:
             cmd.append('--username=' + openerp.tools.config['db_user'])
         if openerp.tools.config['db_host']:
             cmd.append('--host=' + openerp.tools.config['db_host'])
         if openerp.tools.config['db_port']:
             cmd.append('--port=' + str(openerp.tools.config['db_port']))
-        cmd.append(db_name)
+        cmd.append(db)
 
-        stdin, stdout = openerp.tools.exec_pg_command_pipe(*tuple(cmd))
+        if openerp.tools.exec_pg_command(*cmd):
-        stdin.close()
-        data = stdout.read()
-        res = stdout.close()
-
-        if not data or res:
             _logger.error('DUMP DB: %s failed! Please verify the configuration of the database '
                           'password on the server. You may need to create a .pgpass file for '
                           'authentication, or specify `db_password` in the server configuration '
-                          'file.\n %s', db_name, data)
+                          'file.', db)
             raise Exception("Couldn't dump database")
-        _logger.info('DUMP DB successful: %s', db_name)
 
-        return base64.encodestring(data)
+        openerp.tools.osutil.zip_dir(dump_dir, stream, include_dir=False)
 
-def exp_restore(db_name, data):
+    _logger.info('DUMP DB successful: %s', db)
-    with _set_pg_password_in_environment():
-        if exp_db_exist(db_name):
-            _logger.warning('RESTORE DB: %s already exists', db_name)
-            raise Exception("Database already exists")
 
-        _create_empty_database(db_name)
+def exp_restore(db_name, data, copy=False):
+    data_file = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        data_file.write(data.decode('base64'))
+        data_file.close()
+        restore_db(db_name, data_file.name, copy=copy)
+    finally:
+        os.unlink(data_file.name)
+    return True
 
-        cmd = ['pg_restore', '--no-owner']
+@_set_pg_password_in_environment
+def restore_db(db, dump_file, copy=False):
+    assert isinstance(db, basestring)
+    if exp_db_exist(db):
+        _logger.warning('RESTORE DB: %s already exists', db)
+        raise Exception("Database already exists")
+
+    _create_empty_database(db)
+
+    filestore_path = None
+    with openerp.tools.osutil.tempdir() as dump_dir:
+        if zipfile.is_zipfile(dump_file):
+            # v8 format
+            with zipfile.ZipFile(dump_file, 'r') as z:
+                # only extract known members!
+                filestore = [m for m in z.namelist() if m.startswith('filestore/')]
+                z.extractall(dump_dir, ['dump.sql'] + filestore)
+
+                if filestore:
+                    filestore_path = os.path.join(dump_dir, 'filestore')
+
+            pg_cmd = 'psql'
+            pg_args = ['-q', '-f', os.path.join(dump_dir, 'dump.sql')]
+
+        else:
+            # <= 7.0 format (raw pg_dump output)
+            pg_cmd = 'pg_restore'
+            pg_args = ['--no-owner', dump_file]
+
+        args = []
         if openerp.tools.config['db_user']:
-            cmd.append('--username=' + openerp.tools.config['db_user'])
+            args.append('--username=' + openerp.tools.config['db_user'])
         if openerp.tools.config['db_host']:
-            cmd.append('--host=' + openerp.tools.config['db_host'])
+            args.append('--host=' + openerp.tools.config['db_host'])
         if openerp.tools.config['db_port']:
-            cmd.append('--port=' + str(openerp.tools.config['db_port']))
+            args.append('--port=' + str(openerp.tools.config['db_port']))
-        cmd.append('--dbname=' + db_name)
+        args.append('--dbname=' + db)
-        args2 = tuple(cmd)
+        pg_args = args + pg_args
 
-        buf = base64.decodestring(data)
+        if openerp.tools.exec_pg_command(pg_cmd, *pg_args):
-        if os.name == "nt":
-            tmpfile = (os.environ['TMP'] or 'C:\\') + os.tmpnam()
-            file(tmpfile, 'wb').write(buf)
-            args2 = list(args2)
-            args2.append(tmpfile)
-            args2 = tuple(args2)
-        stdin, stdout = openerp.tools.exec_pg_command_pipe(*args2)
-        if not os.name == "nt":
-            stdin.write(base64.decodestring(data))
-        stdin.close()
-        res = stdout.close()
-        if res:
             raise Exception("Couldn't restore database")
-        _logger.info('RESTORE DB: %s', db_name)
 
-        return True
+        registry = openerp.modules.registry.RegistryManager.new(db)
+        with registry.cursor() as cr:
+            if copy:
+                # if it's a copy of a database, force generation of a new dbuuid
+                registry['ir.config_parameter'].init(cr, force=True)
+            if filestore_path:
+                filestore_dest = registry['ir.attachment']._filestore(cr, SUPERUSER_ID)
+                shutil.move(filestore_path, filestore_dest)
+
+            if openerp.tools.config['unaccent']:
+                try:
+                    with cr.savepoint():
+                        cr.execute("CREATE EXTENSION unaccent")
+                except psycopg2.Error:
+                    pass
+
+    _logger.info('RESTORE DB: %s', db)
 
 def exp_rename(old_name, new_name):
     openerp.modules.registry.RegistryManager.delete(old_name)
@@ -280,6 +330,7 @@ def exp_rename(old_name, new_name):
             raise Exception("Couldn't rename database %s to %s: %s" % (old_name, new_name, e))
     return True
 
+@openerp.tools.mute_logger('openerp.sql_db')
 def exp_db_exist(db_name):
     ## Not True: in fact, check if connection to database is possible. The database may exists
     return bool(openerp.sql_db.db_connect(db_name))

openerp/tools/osutil.py

@@ -23,8 +23,12 @@
 Some functions related to the os and os.path module
 """
 
+from contextlib import contextmanager
 import os
 from os.path import join as opj
+import shutil
+import tempfile
+import zipfile
 
 if os.name == 'nt':
     import ctypes
@@ -61,6 +65,30 @@ def walksymlinks(top, topdown=True, onerror=None):
         if not topdown:
             yield dirpath, dirnames, filenames
 
+@contextmanager
+def tempdir():
+    tmpdir = tempfile.mkdtemp()
+    try:
+        yield tmpdir
+    finally:
+        shutil.rmtree(tmpdir)
+
+def zip_dir(path, stream, include_dir=True):      # TODO add ignore list
+    path = os.path.normpath(path)
+    len_prefix = len(os.path.dirname(path)) if include_dir else len(path)
+    if len_prefix:
+        len_prefix += 1
+
+    with zipfile.ZipFile(stream, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as zipf:
+        for dirpath, dirnames, filenames in os.walk(path):
+            for fname in filenames:
+                bname, ext = os.path.splitext(fname)
+                ext = ext or bname
+                if ext not in ['.pyc', '.pyo', '.swp', '.DS_Store']:
+                    path = os.path.normpath(os.path.join(dirpath, fname))
+                    if os.path.isfile(path):
+                        zipf.write(path, path[len_prefix:])
+
 
 if os.name != 'nt':
     getppid = os.getppid