diff --git a/openerp/modules/__init__.py b/openerp/modules/__init__.py index 539abc6553f..cf8d022842e 100644 --- a/openerp/modules/__init__.py +++ b/openerp/modules/__init__.py @@ -52,6 +52,7 @@ import logging import openerp.modules.db import openerp.modules.graph +import openerp.modules.migration logger = netsvc.Logger() @@ -385,152 +386,6 @@ def register_module_classes(m): loaded.append(m) -class MigrationManager(object): - """ - This class manage the migration of modules - Migrations files must be python files containing a "migrate(cr, installed_version)" function. - Theses files must respect a directory tree structure: A 'migrations' folder which containt a - folder by version. Version can be 'module' version or 'server.module' version (in this case, - the files will only be processed by this version of the server). Python file names must start - by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation - Example: - - - `-- migrations - |-- 1.0 - | |-- pre-update_table_x.py - | |-- pre-update_table_y.py - | |-- post-clean-data.py - | `-- README.txt # not processed - |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server - | |-- pre-delete_table_z.py - | `-- post-clean-data.py - `-- foo.py # not processed - - This similar structure is generated by the maintenance module with the migrations files get by - the maintenance contract - - """ - def __init__(self, cr, graph): - self.cr = cr - self.graph = graph - self.migrations = {} - self._get_files() - - def _get_files(self): - - """ - import addons.base.maintenance.utils as maintenance_utils - maintenance_utils.update_migrations_files(self.cr) - #""" - - for pkg in self.graph: - self.migrations[pkg.name] = {} - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): - continue - - self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} - self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} - - def migrate_module(self, pkg, stage): - assert stage in ('pre', 'post') - stageformat = {'pre': '[>%s]', - 'post': '[%s>]', - } - - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): - return - - def convert_version(version): - if version.startswith(release.major_version) and version != release.major_version: - return version # the version number already containt the server version - return "%s.%s" % (release.major_version, version) - - def _get_migration_versions(pkg): - def __get_dir(tree): - return [d for d in tree if tree[d] is not None] - - versions = list(set( - __get_dir(self.migrations[pkg.name]['module']) + - __get_dir(self.migrations[pkg.name]['maintenance']) - )) - versions.sort(key=lambda k: parse_version(convert_version(k))) - return versions - - def _get_migration_files(pkg, version, stage): - """ return a list of tuple (module, file) - """ - m = self.migrations[pkg.name] - lst = [] - - mapping = {'module': opj(pkg.name, 'migrations'), - 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name), - } - - for x in mapping.keys(): - if version in m[x]: - for f in m[x][version]: - if m[x][version][f] is not None: - continue - if not f.startswith(stage + '-'): - continue - lst.append(opj(mapping[x], version, f)) - lst.sort() - return lst - - def mergedict(a, b): - a = a.copy() - a.update(b) - return a - - from openerp.tools.parse_version import parse_version - - parsed_installed_version = parse_version(pkg.installed_version or '') - current_version = parse_version(convert_version(pkg.data['version'])) - - versions = _get_migration_versions(pkg) - - for version in versions: - if parsed_installed_version < parse_version(convert_version(version)) <= current_version: - - strfmt = {'addon': pkg.name, - 'stage': stage, - 'version': stageformat[stage] % version, - } - - for pyfile in _get_migration_files(pkg, version, stage): - name, ext = os.path.splitext(os.path.basename(pyfile)) - if ext.lower() != '.py': - continue - mod = fp = fp2 = None - try: - fp = tools.file_open(pyfile) - - # imp.load_source need a real file object, so we create - # one from the file-like object we get from file_open - fp2 = os.tmpfile() - fp2.write(fp.read()) - fp2.seek(0) - try: - mod = imp.load_source(name, pyfile, fp2) - logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt)) - mod.migrate(self.cr, pkg.installed_version) - except ImportError: - logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt)) - raise - except AttributeError: - logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) - except: - raise - finally: - if fp: - fp.close() - if fp2: - fp2.close() - if mod: - del mod - - def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, report=None): """Migrates+Updates or Installs all module nodes from ``graph`` :param graph: graph of module nodes to load @@ -613,7 +468,7 @@ def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules= processed_modules = [] statusi = 0 pool = pooler.get_pool(cr.dbname) - migrations = MigrationManager(cr, graph) + migrations = openerp.modules.migration.MigrationManager(cr, graph) logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) # register, instanciate and initialize models for each modules diff --git a/openerp/modules/graph.py b/openerp/modules/graph.py index 2a9951f29d9..e6938805266 100644 --- a/openerp/modules/graph.py +++ b/openerp/modules/graph.py @@ -20,6 +20,8 @@ # ############################################################################## +""" Modules dependency graph. """ + import os, sys, imp from os.path import join as opj import itertools diff --git a/openerp/modules/migration.py b/openerp/modules/migration.py new file mode 100644 index 00000000000..0773a192ffb --- /dev/null +++ b/openerp/modules/migration.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Modules migration handling. """ + +import os, sys, imp +from os.path import join as opj +import itertools +import zipimport + +import openerp + +import openerp.osv as osv +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +import openerp.pooler as pooler +from openerp.tools.translate import _ + +import openerp.netsvc as netsvc + +import zipfile +import openerp.release as release + +import re +import base64 +from zipfile import PyZipFile, ZIP_DEFLATED +from cStringIO import StringIO + +import logging + +import openerp.modules.db +import openerp.modules.graph + +logger = netsvc.Logger() + + +class MigrationManager(object): + """ + This class manage the migration of modules + Migrations files must be python files containing a "migrate(cr, installed_version)" function. + Theses files must respect a directory tree structure: A 'migrations' folder which containt a + folder by version. Version can be 'module' version or 'server.module' version (in this case, + the files will only be processed by this version of the server). Python file names must start + by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation + Example: + + + `-- migrations + |-- 1.0 + | |-- pre-update_table_x.py + | |-- pre-update_table_y.py + | |-- post-clean-data.py + | `-- README.txt # not processed + |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server + | |-- pre-delete_table_z.py + | `-- post-clean-data.py + `-- foo.py # not processed + + This similar structure is generated by the maintenance module with the migrations files get by + the maintenance contract + + """ + def __init__(self, cr, graph): + self.cr = cr + self.graph = graph + self.migrations = {} + self._get_files() + + def _get_files(self): + + """ + import addons.base.maintenance.utils as maintenance_utils + maintenance_utils.update_migrations_files(self.cr) + #""" + + for pkg in self.graph: + self.migrations[pkg.name] = {} + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + continue + + get_module_filetree = openerp.modules.get_module_filetree + self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} + self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} + + def migrate_module(self, pkg, stage): + assert stage in ('pre', 'post') + stageformat = {'pre': '[>%s]', + 'post': '[%s>]', + } + + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + return + + def convert_version(version): + if version.startswith(release.major_version) and version != release.major_version: + return version # the version number already containt the server version + return "%s.%s" % (release.major_version, version) + + def _get_migration_versions(pkg): + def __get_dir(tree): + return [d for d in tree if tree[d] is not None] + + versions = list(set( + __get_dir(self.migrations[pkg.name]['module']) + + __get_dir(self.migrations[pkg.name]['maintenance']) + )) + versions.sort(key=lambda k: parse_version(convert_version(k))) + return versions + + def _get_migration_files(pkg, version, stage): + """ return a list of tuple (module, file) + """ + m = self.migrations[pkg.name] + lst = [] + + mapping = {'module': opj(pkg.name, 'migrations'), + 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name), + } + + for x in mapping.keys(): + if version in m[x]: + for f in m[x][version]: + if m[x][version][f] is not None: + continue + if not f.startswith(stage + '-'): + continue + lst.append(opj(mapping[x], version, f)) + lst.sort() + return lst + + def mergedict(a, b): + a = a.copy() + a.update(b) + return a + + from openerp.tools.parse_version import parse_version + + parsed_installed_version = parse_version(pkg.installed_version or '') + current_version = parse_version(convert_version(pkg.data['version'])) + + versions = _get_migration_versions(pkg) + + for version in versions: + if parsed_installed_version < parse_version(convert_version(version)) <= current_version: + + strfmt = {'addon': pkg.name, + 'stage': stage, + 'version': stageformat[stage] % version, + } + + for pyfile in _get_migration_files(pkg, version, stage): + name, ext = os.path.splitext(os.path.basename(pyfile)) + if ext.lower() != '.py': + continue + mod = fp = fp2 = None + try: + fp = tools.file_open(pyfile) + + # imp.load_source need a real file object, so we create + # one from the file-like object we get from file_open + fp2 = os.tmpfile() + fp2.write(fp.read()) + fp2.seek(0) + try: + mod = imp.load_source(name, pyfile, fp2) + logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt)) + mod.migrate(self.cr, pkg.installed_version) + except ImportError: + logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt)) + raise + except AttributeError: + logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) + except: + raise + finally: + if fp: + fp.close() + if fp2: + fp2.close() + if mod: + del mod + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: