2013-01-30 14:56:16 +00:00
# -*- coding: utf-8 -*-
2015-01-19 00:41:08 +00:00
import json
2013-01-30 14:56:16 +00:00
import logging
2013-03-18 11:02:57 +00:00
import os
2014-03-04 17:56:56 +00:00
import shutil
2015-01-19 00:41:08 +00:00
import tempfile
2013-01-30 14:56:16 +00:00
import threading
import traceback
2014-03-04 17:56:56 +00:00
import zipfile
2015-01-19 00:41:08 +00:00
from functools import wraps
from contextlib import closing
2014-03-04 17:56:56 +00:00
import psycopg2
2013-01-30 14:56:16 +00:00
2013-03-27 11:10:14 +00:00
import openerp
2013-01-30 14:56:16 +00:00
from openerp import SUPERUSER_ID
2014-05-30 15:06:16 +00:00
from openerp . exceptions import Warning
2013-03-18 11:02:57 +00:00
import openerp . release
2013-01-30 14:56:16 +00:00
import openerp . sql_db
import openerp . tools
import security
_logger = logging . getLogger ( __name__ )
2014-05-30 15:06:16 +00:00
class DatabaseExists ( Warning ) :
pass
2013-01-30 14:56:16 +00:00
# This should be moved to openerp.modules.db, along side initialize().
def _initialize_db ( id , db_name , demo , lang , user_password ) :
try :
2014-01-09 09:32:58 +00:00
db = openerp . sql_db . db_connect ( db_name )
with closing ( db . cursor ( ) ) as cr :
2014-03-03 16:08:05 +00:00
# TODO this should be removed as it is done by RegistryManager.new().
openerp . modules . db . initialize ( cr )
2013-02-19 16:30:47 +00:00
openerp . tools . config [ ' lang ' ] = lang
cr . commit ( )
2013-01-30 14:56:16 +00:00
2013-03-27 11:10:14 +00:00
registry = openerp . modules . registry . RegistryManager . new (
2014-07-15 10:26:55 +00:00
db_name , demo , None , update_module = True )
2013-01-30 14:56:16 +00:00
2014-01-09 09:32:58 +00:00
with closing ( db . cursor ( ) ) as cr :
2013-02-19 16:30:47 +00:00
if lang :
2013-03-27 11:10:14 +00:00
modobj = registry [ ' ir.module.module ' ]
2013-02-19 16:30:47 +00:00
mids = modobj . search ( cr , SUPERUSER_ID , [ ( ' state ' , ' = ' , ' installed ' ) ] )
modobj . update_translations ( cr , SUPERUSER_ID , mids , lang )
2013-01-30 14:56:16 +00:00
2013-02-19 16:30:47 +00:00
# update admin's password and lang
values = { ' password ' : user_password , ' lang ' : lang }
2013-03-27 11:10:14 +00:00
registry [ ' res.users ' ] . write ( cr , SUPERUSER_ID , [ SUPERUSER_ID ] , values )
2013-01-30 14:56:16 +00:00
2013-02-19 16:30:47 +00:00
cr . execute ( ' SELECT login, password FROM res_users ORDER BY login ' )
cr . commit ( )
2013-01-30 14:56:16 +00:00
except Exception , e :
_logger . exception ( ' CREATE DATABASE failed: ' )
def dispatch ( method , params ) :
2014-03-03 16:08:05 +00:00
if method in [ ' create ' , ' get_progress ' , ' drop ' , ' dump ' , ' restore ' , ' rename ' ,
' change_admin_password ' , ' migrate_databases ' ,
' create_database ' , ' duplicate_database ' ] :
2013-01-30 14:56:16 +00:00
passwd = params [ 0 ]
params = params [ 1 : ]
security . check_super ( passwd )
2014-03-03 16:08:05 +00:00
elif method in [ ' db_exist ' , ' list ' , ' list_lang ' , ' server_version ' ] :
2013-01-30 14:56:16 +00:00
# params = params
# No security check for these methods
pass
else :
raise KeyError ( " Method not found: %s " % method )
fn = globals ( ) [ ' exp_ ' + method ]
return fn ( * params )
def _create_empty_database ( name ) :
db = openerp . sql_db . db_connect ( ' postgres ' )
2014-01-09 09:32:58 +00:00
with closing ( db . cursor ( ) ) as cr :
chosen_template = openerp . tools . config [ ' db_template ' ]
cr . execute ( " SELECT datname FROM pg_database WHERE datname = %s " ,
( name , ) )
if cr . fetchall ( ) :
2014-05-30 15:06:16 +00:00
raise DatabaseExists ( " database %r already exists! " % ( name , ) )
2014-01-09 09:32:58 +00:00
else :
2014-03-03 16:08:05 +00:00
cr . autocommit ( True ) # avoid transaction block
2014-01-09 09:32:58 +00:00
cr . execute ( """ CREATE DATABASE " %s " ENCODING ' unicode ' TEMPLATE " %s " """ % ( name , chosen_template ) )
2013-01-30 14:56:16 +00:00
def exp_create_database ( db_name , demo , lang , user_password = ' admin ' ) :
""" Similar to exp_create but blocking. """
_logger . info ( ' Create database ` %s `. ' , db_name )
_create_empty_database ( db_name )
_initialize_db ( id , db_name , demo , lang , user_password )
return True
def exp_duplicate_database ( db_original_name , db_name ) :
_logger . info ( ' Duplicate database ` %s ` to ` %s `. ' , db_original_name , db_name )
openerp . sql_db . close_db ( db_original_name )
db = openerp . sql_db . db_connect ( ' postgres ' )
2014-01-09 09:32:58 +00:00
with closing ( db . cursor ( ) ) as cr :
2014-03-03 16:08:05 +00:00
cr . autocommit ( True ) # avoid transaction block
2015-03-18 17:20:25 +00:00
_drop_conn ( cr , db_original_name )
2013-01-30 14:56:16 +00:00
cr . execute ( """ CREATE DATABASE " %s " ENCODING ' unicode ' TEMPLATE " %s " """ % ( db_name , db_original_name ) )
2014-03-21 15:56:59 +00:00
from_fs = openerp . tools . config . filestore ( db_original_name )
to_fs = openerp . tools . config . filestore ( db_name )
if os . path . exists ( from_fs ) and not os . path . exists ( to_fs ) :
2014-06-28 14:37:29 +00:00
shutil . copytree ( from_fs , to_fs )
2013-01-30 14:56:16 +00:00
return True
2014-06-26 11:41:29 +00:00
def _drop_conn ( cr , db_name ) :
# Try to terminate all other connections that might prevent
# dropping the database
try :
# PostgreSQL 9.2 renamed pg_stat_activity.procpid to pid:
# http://www.postgresql.org/docs/9.2/static/release-9-2.html#AEN110389
pid_col = ' pid ' if cr . _cnx . server_version > = 90200 else ' procpid '
cr . execute ( """ SELECT pg_terminate_backend( %(pid_col)s )
FROM pg_stat_activity
WHERE datname = % % s AND
% ( pid_col ) s != pg_backend_pid ( ) """ % { ' pid_col ' : pid_col},
( db_name , ) )
except Exception :
pass
2013-01-30 14:56:16 +00:00
def exp_drop ( db_name ) :
2014-02-06 11:02:20 +00:00
if db_name not in exp_list ( True ) :
2013-01-30 14:56:16 +00:00
return False
openerp . modules . registry . RegistryManager . delete ( db_name )
openerp . sql_db . close_db ( db_name )
db = openerp . sql_db . db_connect ( ' postgres ' )
2014-01-09 09:32:58 +00:00
with closing ( db . cursor ( ) ) as cr :
cr . autocommit ( True ) # avoid transaction block
2014-06-26 11:41:29 +00:00
_drop_conn ( cr , db_name )
2013-01-30 14:56:16 +00:00
try :
cr . execute ( ' DROP DATABASE " %s " ' % db_name )
except Exception , e :
_logger . error ( ' DROP DB: %s failed: \n %s ' , db_name , e )
raise Exception ( " Couldn ' t drop database %s : %s " % ( db_name , e ) )
else :
_logger . info ( ' DROP DB: %s ' , db_name )
2014-03-21 15:56:59 +00:00
fs = openerp . tools . config . filestore ( db_name )
if os . path . exists ( fs ) :
shutil . rmtree ( fs )
2013-01-30 14:56:16 +00:00
return True
def exp_dump ( db_name ) :
2014-03-04 17:56:56 +00:00
with tempfile . TemporaryFile ( ) as t :
dump_db ( db_name , t )
t . seek ( 0 )
return t . read ( ) . encode ( ' base64 ' )
2015-01-19 00:41:08 +00:00
def dump_db_manifest ( cr ) :
pg_version = " %d . %d " % divmod ( cr . _obj . connection . server_version / 100 , 100 )
2015-03-16 15:17:59 +00:00
cr . execute ( " SELECT name, latest_version FROM ir_module_module WHERE state = ' installed ' " )
modules = dict ( cr . fetchall ( ) )
2015-01-19 00:41:08 +00:00
manifest = {
' odoo_dump ' : ' 1 ' ,
' db_name ' : cr . dbname ,
' version ' : openerp . release . version ,
' version_info ' : openerp . release . version_info ,
' major_version ' : openerp . release . major_version ,
' pg_version ' : pg_version ,
' modules ' : modules ,
}
return manifest
def dump_db ( db_name , stream , backup_format = ' zip ' ) :
2015-01-19 01:42:34 +00:00
""" Dump database `db` into file-like object `stream` if stream is None
return a file object with the dump """
_logger . info ( ' DUMP DB: %s format %s ' , db_name , backup_format )
2014-03-04 17:56:56 +00:00
2015-01-19 00:41:08 +00:00
cmd = [ ' pg_dump ' , ' --no-owner ' ]
if openerp . tools . config [ ' db_user ' ] :
cmd . append ( ' --username= ' + openerp . tools . config [ ' db_user ' ] )
if openerp . tools . config [ ' db_host ' ] :
cmd . append ( ' --host= ' + openerp . tools . config [ ' db_host ' ] )
if openerp . tools . config [ ' db_port ' ] :
cmd . append ( ' --port= ' + str ( openerp . tools . config [ ' db_port ' ] ) )
cmd . append ( db_name )
if backup_format == ' zip ' :
with openerp . tools . osutil . tempdir ( ) as dump_dir :
2015-03-16 15:17:59 +00:00
filestore = openerp . tools . config . filestore ( db_name )
if os . path . exists ( filestore ) :
shutil . copytree ( filestore , os . path . join ( dump_dir , ' filestore ' ) )
with open ( os . path . join ( dump_dir , ' manifest.json ' ) , ' w ' ) as fh :
db = openerp . sql_db . db_connect ( db_name )
with db . cursor ( ) as cr :
json . dump ( dump_db_manifest ( cr ) , fh , indent = 4 )
2015-01-19 00:41:08 +00:00
cmd . insert ( - 1 , ' --file= ' + os . path . join ( dump_dir , ' dump.sql ' ) )
openerp . tools . exec_pg_command ( * cmd )
2015-01-19 01:42:34 +00:00
if stream :
2015-03-09 05:18:20 +00:00
openerp . tools . osutil . zip_dir ( dump_dir , stream , include_dir = False , fnct_sort = lambda file_name : file_name != ' dump.sql ' )
2015-01-19 01:42:34 +00:00
else :
t = tempfile . TemporaryFile ( )
2015-03-09 05:18:20 +00:00
openerp . tools . osutil . zip_dir ( dump_dir , t , include_dir = False , fnct_sort = lambda file_name : file_name != ' dump.sql ' )
2015-01-19 01:42:34 +00:00
t . seek ( 0 )
return t
2015-01-19 00:41:08 +00:00
else :
cmd . insert ( - 1 , ' --format=c ' )
stdin , stdout = openerp . tools . exec_pg_command_pipe ( * cmd )
2015-01-19 01:42:34 +00:00
if stream :
shutil . copyfileobj ( stdout , stream )
else :
return stdout
2014-03-04 17:56:56 +00:00
def exp_restore ( db_name , data , copy = False ) :
data_file = tempfile . NamedTemporaryFile ( delete = False )
try :
data_file . write ( data . decode ( ' base64 ' ) )
data_file . close ( )
restore_db ( db_name , data_file . name , copy = copy )
finally :
os . unlink ( data_file . name )
return True
def restore_db ( db , dump_file , copy = False ) :
assert isinstance ( db , basestring )
if exp_db_exist ( db ) :
_logger . warning ( ' RESTORE DB: %s already exists ' , db )
raise Exception ( " Database already exists " )
_create_empty_database ( db )
2013-01-30 14:56:16 +00:00
2014-03-04 17:56:56 +00:00
filestore_path = None
with openerp . tools . osutil . tempdir ( ) as dump_dir :
if zipfile . is_zipfile ( dump_file ) :
# v8 format
with zipfile . ZipFile ( dump_file , ' r ' ) as z :
# only extract known members!
filestore = [ m for m in z . namelist ( ) if m . startswith ( ' filestore/ ' ) ]
z . extractall ( dump_dir , [ ' dump.sql ' ] + filestore )
2013-01-30 14:56:16 +00:00
2014-03-04 17:56:56 +00:00
if filestore :
filestore_path = os . path . join ( dump_dir , ' filestore ' )
pg_cmd = ' psql '
pg_args = [ ' -q ' , ' -f ' , os . path . join ( dump_dir , ' dump.sql ' ) ]
else :
# <= 7.0 format (raw pg_dump output)
pg_cmd = ' pg_restore '
pg_args = [ ' --no-owner ' , dump_file ]
2013-01-30 14:56:16 +00:00
2014-03-04 17:56:56 +00:00
args = [ ]
2013-01-30 14:56:16 +00:00
if openerp . tools . config [ ' db_user ' ] :
2014-03-04 17:56:56 +00:00
args . append ( ' --username= ' + openerp . tools . config [ ' db_user ' ] )
2013-01-30 14:56:16 +00:00
if openerp . tools . config [ ' db_host ' ] :
2014-03-04 17:56:56 +00:00
args . append ( ' --host= ' + openerp . tools . config [ ' db_host ' ] )
2013-01-30 14:56:16 +00:00
if openerp . tools . config [ ' db_port ' ] :
2014-03-04 17:56:56 +00:00
args . append ( ' --port= ' + str ( openerp . tools . config [ ' db_port ' ] ) )
args . append ( ' --dbname= ' + db )
pg_args = args + pg_args
if openerp . tools . exec_pg_command ( pg_cmd , * pg_args ) :
2014-03-03 16:08:05 +00:00
raise Exception ( " Couldn ' t restore database " )
2013-01-30 14:56:16 +00:00
2014-03-04 17:56:56 +00:00
registry = openerp . modules . registry . RegistryManager . new ( db )
with registry . cursor ( ) as cr :
if copy :
# if it's a copy of a database, force generation of a new dbuuid
registry [ ' ir.config_parameter ' ] . init ( cr , force = True )
if filestore_path :
filestore_dest = registry [ ' ir.attachment ' ] . _filestore ( cr , SUPERUSER_ID )
shutil . move ( filestore_path , filestore_dest )
if openerp . tools . config [ ' unaccent ' ] :
try :
with cr . savepoint ( ) :
cr . execute ( " CREATE EXTENSION unaccent " )
except psycopg2 . Error :
pass
_logger . info ( ' RESTORE DB: %s ' , db )
2013-01-30 14:56:16 +00:00
def exp_rename ( old_name , new_name ) :
openerp . modules . registry . RegistryManager . delete ( old_name )
openerp . sql_db . close_db ( old_name )
db = openerp . sql_db . db_connect ( ' postgres ' )
2014-01-09 09:32:58 +00:00
with closing ( db . cursor ( ) ) as cr :
2014-03-03 16:08:05 +00:00
cr . autocommit ( True ) # avoid transaction block
2014-06-26 11:41:29 +00:00
_drop_conn ( cr , old_name )
2013-01-30 14:56:16 +00:00
try :
cr . execute ( ' ALTER DATABASE " %s " RENAME TO " %s " ' % ( old_name , new_name ) )
2014-01-16 18:54:15 +00:00
_logger . info ( ' RENAME DB: %s -> %s ' , old_name , new_name )
2013-01-30 14:56:16 +00:00
except Exception , e :
_logger . error ( ' RENAME DB: %s -> %s failed: \n %s ' , old_name , new_name , e )
raise Exception ( " Couldn ' t rename database %s to %s : %s " % ( old_name , new_name , e ) )
2014-03-21 15:56:59 +00:00
old_fs = openerp . tools . config . filestore ( old_name )
new_fs = openerp . tools . config . filestore ( new_name )
if os . path . exists ( old_fs ) and not os . path . exists ( new_fs ) :
shutil . move ( old_fs , new_fs )
2013-01-30 14:56:16 +00:00
return True
2014-03-04 17:56:56 +00:00
@openerp.tools.mute_logger ( ' openerp.sql_db ' )
2013-01-30 14:56:16 +00:00
def exp_db_exist ( db_name ) :
## Not True: in fact, check if connection to database is possible. The database may exists
return bool ( openerp . sql_db . db_connect ( db_name ) )
def exp_list ( document = False ) :
if not openerp . tools . config [ ' list_db ' ] and not document :
raise openerp . exceptions . AccessDenied ( )
chosen_template = openerp . tools . config [ ' db_template ' ]
templates_list = tuple ( set ( [ ' template0 ' , ' template1 ' , ' postgres ' , chosen_template ] ) )
db = openerp . sql_db . db_connect ( ' postgres ' )
2014-01-09 09:32:58 +00:00
with closing ( db . cursor ( ) ) as cr :
2013-01-30 14:56:16 +00:00
try :
db_user = openerp . tools . config [ " db_user " ]
if not db_user and os . name == ' posix ' :
import pwd
db_user = pwd . getpwuid ( os . getuid ( ) ) [ 0 ]
if not db_user :
cr . execute ( " select usename from pg_user where usesysid=(select datdba from pg_database where datname= %s ) " , ( openerp . tools . config [ " db_name " ] , ) )
res = cr . fetchone ( )
db_user = res and str ( res [ 0 ] )
if db_user :
cr . execute ( " select datname from pg_database where datdba=(select usesysid from pg_user where usename= %s ) and datname not in %s order by datname " , ( db_user , templates_list ) )
else :
cr . execute ( " select datname from pg_database where datname not in %s order by datname " , ( templates_list , ) )
2014-02-20 13:25:48 +00:00
res = [ openerp . tools . ustr ( name ) for ( name , ) in cr . fetchall ( ) ]
2013-01-30 14:56:16 +00:00
except Exception :
res = [ ]
res . sort ( )
return res
def exp_change_admin_password ( new_password ) :
openerp . tools . config [ ' admin_passwd ' ] = new_password
openerp . tools . config . save ( )
return True
def exp_list_lang ( ) :
return openerp . tools . scan_languages ( )
def exp_server_version ( ) :
""" Return the version of the server
Used by the client to verify the compatibility with its own version
"""
2013-03-18 11:02:57 +00:00
return openerp . release . version
2013-01-30 14:56:16 +00:00
def exp_migrate_databases ( databases ) :
for db in databases :
_logger . info ( ' migrate database %s ' , db )
openerp . tools . config [ ' update ' ] [ ' base ' ] = True
2013-03-27 11:10:14 +00:00
openerp . modules . registry . RegistryManager . new ( db , force_demo = False , update_module = True )
2013-01-30 14:56:16 +00:00
return True