diff --git a/openerp/__init__.py b/openerp/__init__.py index c5f7732edd0..02d16401293 100644 --- a/openerp/__init__.py +++ b/openerp/__init__.py @@ -22,6 +22,7 @@ import addons import ir import loglevels +import modules import netsvc import osv import pooler diff --git a/openerp/addons/__init__.py b/openerp/addons/__init__.py index c2c6001e04a..d019985f34c 100644 --- a/openerp/addons/__init__.py +++ b/openerp/addons/__init__.py @@ -20,966 +20,17 @@ # ############################################################################## -import os, sys, imp -from os.path import join as opj -import itertools -import zipimport +""" Addons module. -import openerp.osv as osv -import openerp.tools as tools -import openerp.tools.osutil as osutil -from openerp.tools.safe_eval import safe_eval as eval -import openerp.pooler as pooler -from openerp.tools.translate import _ +This module only serves to contain OpenERP addons. For the code to +manage those addons, see openerp.modules. This module conveniently +reexports some symbols from openerp.modules. Importing them from here +is deprecated. -import openerp.netsvc as netsvc - -import zipfile -import openerp.release as release - -import re -import base64 -from zipfile import PyZipFile, ZIP_DEFLATED -from cStringIO import StringIO - -import logging - -logger = netsvc.Logger() - -_ad = os.path.dirname(__file__) # default addons path (base) -ad_paths = [] - -# Modules already loaded -loaded = [] - -def initialize_sys_path(): - global ad_paths - - if ad_paths: - return - - ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) - - sys.path.insert(1, _ad) - - ad_cnt=1 - for adp in ad_paths: - if adp != _ad: - sys.path.insert(ad_cnt, adp) - ad_cnt+=1 - - ad_paths.append(_ad) # for get_module_path - -class Graph(dict): - - def addNode(self, name, deps): - max_depth, father = 0, None - for n in [Node(x, self) for x in deps]: - if n.depth >= max_depth: - father = n - max_depth = n.depth - if father: - father.addChild(name) - else: - Node(name, self) - - def update_from_db(self, cr): - if not len(self): - return - # update the graph with values from the database (if exist) - ## First, we set the default values for each package in graph - additional_data = dict.fromkeys(self.keys(), {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None}) - ## Then we get the values from the database - cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' - ' FROM ir_module_module' - ' WHERE name IN %s',(tuple(additional_data),) - ) - - ## and we update the default values with values from the database - additional_data.update(dict([(x.pop('name'), x) for x in cr.dictfetchall()])) - - for package in self.values(): - for k, v in additional_data[package.name].items(): - setattr(package, k, v) - - def __iter__(self): - level = 0 - done = set(self.keys()) - while done: - level_modules = [(name, module) for name, module in self.items() if module.depth==level] - for name, module in level_modules: - done.remove(name) - yield module - level += 1 - -class Singleton(object): - def __new__(cls, name, graph): - if name in graph: - inst = graph[name] - else: - inst = object.__new__(cls) - inst.name = name - graph[name] = inst - return inst - - -class Node(Singleton): - - def __init__(self, name, graph): - self.graph = graph - if not hasattr(self, 'children'): - self.children = [] - if not hasattr(self, 'depth'): - self.depth = 0 - - def addChild(self, name): - node = Node(name, self.graph) - node.depth = self.depth + 1 - if node not in self.children: - self.children.append(node) - for attr in ('init', 'update', 'demo'): - if hasattr(self, attr): - setattr(node, attr, True) - self.children.sort(lambda x, y: cmp(x.name, y.name)) - - def __setattr__(self, name, value): - super(Singleton, self).__setattr__(name, value) - if name in ('init', 'update', 'demo'): - tools.config[name][self.name] = 1 - for child in self.children: - setattr(child, name, value) - if name == 'depth': - for child in self.children: - setattr(child, name, value + 1) - - def __iter__(self): - return itertools.chain(iter(self.children), *map(iter, self.children)) - - def __str__(self): - return self._pprint() - - def _pprint(self, depth=0): - s = '%s\n' % self.name - for c in self.children: - s += '%s`-> %s' % (' ' * depth, c._pprint(depth+1)) - return s - - -def get_module_path(module, downloaded=False): - """Return the path of the given module. - - Search the addons paths and return the first path where the given - module is found. If downloaded is True, return the default addons - path if nothing else is found. - - """ - initialize_sys_path() - for adp in ad_paths: - if os.path.exists(opj(adp, module)) or os.path.exists(opj(adp, '%s.zip' % module)): - return opj(adp, module) - - if downloaded: - return opj(_ad, module) - logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: module not found' % (module,)) - return False - - -def get_module_filetree(module, dir='.'): - path = get_module_path(module) - if not path: - return False - - dir = os.path.normpath(dir) - if dir == '.': - dir = '' - if dir.startswith('..') or (dir and dir[0] == '/'): - raise Exception('Cannot access file outside the module') - - if not os.path.isdir(path): - # zipmodule - zip = zipfile.ZipFile(path + ".zip") - files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] - else: - files = osutil.listdir(path, True) - - tree = {} - for f in files: - if not f.startswith(dir): - continue - - if dir: - f = f[len(dir)+int(not dir.endswith('/')):] - lst = f.split(os.sep) - current = tree - while len(lst) != 1: - current = current.setdefault(lst.pop(0), {}) - current[lst.pop(0)] = None - - return tree - -def zip_directory(directory, b64enc=True, src=True): - """Compress a directory - - @param directory: The directory to compress - @param base64enc: if True the function will encode the zip file with base64 - @param src: Integrate the source files - - @return: a string containing the zip file - """ - - RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I) - - def _zippy(archive, path, src=True): - path = os.path.abspath(path) - base = os.path.basename(path) - for f in osutil.listdir(path, True): - bf = os.path.basename(f) - if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')): - archive.write(os.path.join(path, f), os.path.join(base, f)) - - archname = StringIO() - archive = PyZipFile(archname, "w", ZIP_DEFLATED) - - # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8) - directory = tools.ustr(directory).encode('utf-8') - - archive.writepy(directory) - _zippy(archive, directory, src=src) - archive.close() - archive_data = archname.getvalue() - archname.close() - - if b64enc: - return base64.encodestring(archive_data) - - return archive_data - -def get_module_as_zip(modulename, b64enc=True, src=True): - """Generate a module as zip file with the source or not and can do a base64 encoding - - @param modulename: The module name - @param b64enc: if True the function will encode the zip file with base64 - @param src: Integrate the source files - - @return: a stream to store in a file-like object - """ - - ap = get_module_path(str(modulename)) - if not ap: - raise Exception('Unable to find path for module %s' % modulename) - - ap = ap.encode('utf8') - if os.path.isfile(ap + '.zip'): - val = file(ap + '.zip', 'rb').read() - if b64enc: - val = base64.encodestring(val) - else: - val = zip_directory(ap, b64enc, src) - - return val - - -def get_module_resource(module, *args): - """Return the full path of a resource of the given module. - - @param module: the module - @param args: the resource path components - - @return: absolute path to the resource - """ - a = get_module_path(module) - if not a: return False - resource_path = opj(a, *args) - if zipfile.is_zipfile( a +'.zip') : - zip = zipfile.ZipFile( a + ".zip") - files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] - resource_path = '/'.join(args) - if resource_path in files: - return opj(a, resource_path) - elif os.path.exists(resource_path): - return resource_path - return False - -def get_modules(): - """Returns the list of module names - """ - def listdir(dir): - def clean(name): - name = os.path.basename(name) - if name[-4:] == '.zip': - name = name[:-4] - return name - - def is_really_module(name): - name = opj(dir, name) - return os.path.isdir(name) or zipfile.is_zipfile(name) - return map(clean, filter(is_really_module, os.listdir(dir))) - - plist = [] - initialize_sys_path() - for ad in ad_paths: - plist.extend(listdir(ad)) - return list(set(plist)) - -def load_information_from_description_file(module): - """ - :param module: The name of the module (sale, purchase, ...) - """ - - for filename in ['__openerp__.py', '__terp__.py']: - description_file = get_module_resource(module, filename) - if description_file : - desc_f = tools.file_open(description_file) - try: - return eval(desc_f.read()) - finally: - desc_f.close() - - #TODO: refactor the logger in this file to follow the logging guidelines - # for 6.0 - logging.getLogger('addons').debug('The module %s does not contain a description file:'\ - '__openerp__.py or __terp__.py (deprecated)', module) - return {} - -def get_modules_with_version(): - modules = get_modules() - res = {} - for module in modules: - try: - info = load_information_from_description_file(module) - res[module] = "%s.%s" % (release.major_version, info['version']) - except Exception, e: - continue - return res - -def create_graph(cr, module_list, force=None): - graph = Graph() - upgrade_graph(graph, cr, module_list, force) - return graph - -def upgrade_graph(graph, cr, module_list, force=None): - if force is None: - force = [] - packages = [] - len_graph = len(graph) - for module in module_list: - mod_path = get_module_path(module) - terp_file = get_module_resource(module, '__openerp__.py') - if not terp_file: - terp_file = get_module_resource(module, '__terp__.py') - if not mod_path or not terp_file: - logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not found, skipped' % (module)) - continue - - if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'): - terp_f = tools.file_open(terp_file) - try: - info = eval(terp_f.read()) - except Exception: - logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: eval file %s' % (module, terp_file)) - raise - finally: - terp_f.close() - if info.get('installable', True): - packages.append((module, info.get('depends', []), info)) - else: - logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not installable, skipped' % (module)) - - dependencies = dict([(p, deps) for p, deps, data in packages]) - current, later = set([p for p, dep, data in packages]), set() - - while packages and current > later: - package, deps, data = packages[0] - - # if all dependencies of 'package' are already in the graph, add 'package' in the graph - if reduce(lambda x, y: x and y in graph, deps, True): - if not package in current: - packages.pop(0) - continue - later.clear() - current.remove(package) - graph.addNode(package, deps) - node = Node(package, graph) - node.data = data - for kind in ('init', 'demo', 'update'): - if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: - setattr(node, kind, True) - else: - later.add(package) - packages.append((package, deps, data)) - packages.pop(0) - - graph.update_from_db(cr) - - for package in later: - unmet_deps = filter(lambda p: p not in graph, dependencies[package]) - logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: Unmet dependencies: %s' % (package, ', '.join(unmet_deps))) - - result = len(graph) - len_graph - if result != len(module_list): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'Not all modules have loaded.') - return result - - -def init_module_objects(cr, module_name, obj_list): - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: creating or updating database tables' % module_name) - todo = [] - for obj in obj_list: - try: - result = obj._auto_init(cr, {'module': module_name}) - except Exception, e: - raise - if result: - todo += result - if hasattr(obj, 'init'): - obj.init(cr) - cr.commit() - todo.sort() - for t in todo: - t[1](cr, *t[2]) - cr.commit() - - -def register_class(m): - """ - Register module named m, if not already registered - """ - - def log(e): - mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or '' - msg = "Couldn't load %smodule %s" % (mt, m) - logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg) - logger.notifyChannel('init', netsvc.LOG_CRITICAL, e) - - global loaded - if m in loaded: - return - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m) - mod_path = get_module_path(m) - - initialize_sys_path() - try: - zip_mod_path = mod_path + '.zip' - if not os.path.isfile(zip_mod_path): - fm = imp.find_module(m, ad_paths) - try: - imp.load_module(m, *fm) - finally: - if fm[0]: - fm[0].close() - else: - zimp = zipimport.zipimporter(zip_mod_path) - zimp.load_module(m) - except Exception, e: - log(e) - raise - else: - loaded.append(m) - - -class MigrationManager(object): - """ - This class manage the migration of modules - Migrations files must be python files containing a "migrate(cr, installed_version)" function. - Theses files must respect a directory tree structure: A 'migrations' folder which containt a - folder by version. Version can be 'module' version or 'server.module' version (in this case, - the files will only be processed by this version of the server). Python file names must start - by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation - Example: - - - `-- migrations - |-- 1.0 - | |-- pre-update_table_x.py - | |-- pre-update_table_y.py - | |-- post-clean-data.py - | `-- README.txt # not processed - |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server - | |-- pre-delete_table_z.py - | `-- post-clean-data.py - `-- foo.py # not processed - - This similar structure is generated by the maintenance module with the migrations files get by - the maintenance contract - - """ - def __init__(self, cr, graph): - self.cr = cr - self.graph = graph - self.migrations = {} - self._get_files() - - def _get_files(self): - - """ - import addons.base.maintenance.utils as maintenance_utils - maintenance_utils.update_migrations_files(self.cr) - #""" - - for pkg in self.graph: - self.migrations[pkg.name] = {} - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): - continue - - self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} - self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} - - def migrate_module(self, pkg, stage): - assert stage in ('pre', 'post') - stageformat = {'pre': '[>%s]', - 'post': '[%s>]', - } - - if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): - return - - def convert_version(version): - if version.startswith(release.major_version) and version != release.major_version: - return version # the version number already containt the server version - return "%s.%s" % (release.major_version, version) - - def _get_migration_versions(pkg): - def __get_dir(tree): - return [d for d in tree if tree[d] is not None] - - versions = list(set( - __get_dir(self.migrations[pkg.name]['module']) + - __get_dir(self.migrations[pkg.name]['maintenance']) - )) - versions.sort(key=lambda k: parse_version(convert_version(k))) - return versions - - def _get_migration_files(pkg, version, stage): - """ return a list of tuple (module, file) - """ - m = self.migrations[pkg.name] - lst = [] - - mapping = {'module': opj(pkg.name, 'migrations'), - 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name), - } - - for x in mapping.keys(): - if version in m[x]: - for f in m[x][version]: - if m[x][version][f] is not None: - continue - if not f.startswith(stage + '-'): - continue - lst.append(opj(mapping[x], version, f)) - lst.sort() - return lst - - def mergedict(a, b): - a = a.copy() - a.update(b) - return a - - from openerp.tools.parse_version import parse_version - - parsed_installed_version = parse_version(pkg.installed_version or '') - current_version = parse_version(convert_version(pkg.data.get('version', '0'))) - - versions = _get_migration_versions(pkg) - - for version in versions: - if parsed_installed_version < parse_version(convert_version(version)) <= current_version: - - strfmt = {'addon': pkg.name, - 'stage': stage, - 'version': stageformat[stage] % version, - } - - for pyfile in _get_migration_files(pkg, version, stage): - name, ext = os.path.splitext(os.path.basename(pyfile)) - if ext.lower() != '.py': - continue - mod = fp = fp2 = None - try: - fp = tools.file_open(pyfile) - - # imp.load_source need a real file object, so we create - # one from the file-like object we get from file_open - fp2 = os.tmpfile() - fp2.write(fp.read()) - fp2.seek(0) - try: - mod = imp.load_source(name, pyfile, fp2) - logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt)) - mod.migrate(self.cr, pkg.installed_version) - except ImportError: - logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt)) - raise - except AttributeError: - logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) - except: - raise - finally: - if fp: - fp.close() - if fp2: - fp2.close() - if mod: - del mod - -log = logging.getLogger('init') - -def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, **kwargs): - """Migrates+Updates or Installs all module nodes from ``graph`` - :param graph: graph of module nodes to load - :param status: status dictionary for keeping track of progress - :param perform_checks: whether module descriptors should be checked for validity (prints warnings - for same cases, and even raise osv_except if certificate is invalid) - :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped - :return: list of modules that were installed or updated - """ - def process_sql_file(cr, fp): - queries = fp.read().split(';') - for query in queries: - new_query = ' '.join(query.split()) - if new_query: - cr.execute(new_query) - - def load_init_update_xml(cr, m, idref, mode, kind): - for filename in package.data.get('%s_xml' % kind, []): - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, filename)) - _, ext = os.path.splitext(filename) - fp = tools.file_open(opj(m, filename)) - try: - if ext == '.csv': - noupdate = (kind == 'init') - tools.convert_csv_import(cr, m, os.path.basename(filename), fp.read(), idref, mode=mode, noupdate=noupdate) - elif ext == '.sql': - process_sql_file(cr, fp) - elif ext == '.yml': - tools.convert_yaml_import(cr, m, fp, idref, mode=mode, **kwargs) - else: - tools.convert_xml_import(cr, m, fp, idref, mode=mode, **kwargs) - finally: - fp.close() - - def load_demo_xml(cr, m, idref, mode): - for xml in package.data.get('demo_xml', []): - name, ext = os.path.splitext(xml) - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, xml)) - fp = tools.file_open(opj(m, xml)) - try: - if ext == '.csv': - tools.convert_csv_import(cr, m, os.path.basename(xml), fp.read(), idref, mode=mode, noupdate=True) - elif ext == '.yml': - tools.convert_yaml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) - else: - tools.convert_xml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) - finally: - fp.close() - - def load_data(cr, module_name, id_map, mode): - _load_data(cr, module_name, id_map, mode, 'data') - - def load_demo(cr, module_name, id_map, mode): - _load_data(cr, module_name, id_map, mode, 'demo') - - def load_test(cr, module_name, id_map, mode): - cr.commit() - if not tools.config.options['test_disable']: - try: - _load_data(cr, module_name, id_map, mode, 'test') - except Exception, e: - logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) - finally: - if tools.config.options['test_commit']: - cr.commit() - else: - cr.rollback() - - def _load_data(cr, module_name, id_map, mode, kind): - for filename in package.data.get(kind, []): - noupdate = (kind == 'demo') - _, ext = os.path.splitext(filename) - log.info("module %s: loading %s", module_name, filename) - pathname = os.path.join(module_name, filename) - file = tools.file_open(pathname) - try: - if ext == '.sql': - process_sql_file(cr, file) - elif ext == '.csv': - noupdate = (kind == 'init') - tools.convert_csv_import(cr, module_name, pathname, file.read(), id_map, mode, noupdate) - elif ext == '.yml': - tools.convert_yaml_import(cr, module_name, file, id_map, mode, noupdate) - else: - tools.convert_xml_import(cr, module_name, file, id_map, mode, noupdate) - finally: - file.close() - - # **kwargs is passed directly to convert_xml_import - if not status: - status = {} - - status = status.copy() - processed_modules = [] - statusi = 0 - pool = pooler.get_pool(cr.dbname) - migrations = MigrationManager(cr, graph) - modobj = None - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) - - for package in graph: - if skip_modules and package.name in skip_modules: - continue - logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) - migrations.migrate_module(package, 'pre') - register_class(package.name) - modules = pool.instanciate(package.name, cr) - if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): - init_module_objects(cr, package.name, modules) - cr.commit() - - for package in graph: - status['progress'] = (float(statusi)+0.1) / len(graph) - m = package.name - mid = package.id - - if skip_modules and m in skip_modules: - continue - - if modobj is None: - modobj = pool.get('ir.module.module') - - if modobj and perform_checks: - modobj.check(cr, 1, [mid]) - - idref = {} - status['progress'] = (float(statusi)+0.4) / len(graph) - - mode = 'update' - if hasattr(package, 'init') or package.state == 'to install': - mode = 'init' - - if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): - for kind in ('init', 'update'): - if package.state=='to upgrade': - # upgrading the module information - modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data)) - load_init_update_xml(cr, m, idref, mode, kind) - load_data(cr, m, idref, mode) - if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): - status['progress'] = (float(statusi)+0.75) / len(graph) - load_demo_xml(cr, m, idref, mode) - load_demo(cr, m, idref, mode) - cr.execute('update ir_module_module set demo=%s where id=%s', (True, mid)) - - # launch tests only in demo mode, as most tests will depend - # on demo data. Other tests can be added into the regular - # 'data' section, but should probably not alter the data, - # as there is no rollback. - load_test(cr, m, idref, mode) - - processed_modules.append(package.name) - - migrations.migrate_module(package, 'post') - - if modobj: - ver = release.major_version + '.' + package.data.get('version', '1.0') - # Set new modules and dependencies - modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver}) - cr.commit() - # Update translations for all installed languages - modobj.update_translations(cr, 1, [mid], None, {'overwrite': tools.config['overwrite_existing_translations']}) - cr.commit() - - package.state = 'installed' - for kind in ('init', 'demo', 'update'): - if hasattr(package, kind): - delattr(package, kind) - - statusi += 1 - - cr.commit() - - return processed_modules - -def _check_module_names(cr, module_names): - mod_names = set(module_names) - if 'base' in mod_names: - # ignore dummy 'all' module - if 'all' in mod_names: - mod_names.remove('all') - if mod_names: - cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),)) - if cr.dictfetchone()['count'] != len(mod_names): - # find out what module name(s) are incorrect: - cr.execute("SELECT name FROM ir_module_module") - incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) - logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) - -def load_modules(db, force_demo=False, status=None, update_module=False): - - initialize_sys_path() - - # Backward compatibility: addons don't have to import openerp.xxx, they still can import xxx - for k, v in list(sys.modules.items()): - if k.startswith('openerp.') and sys.modules.get(k[8:]) is None: - sys.modules[k[8:]] = v - - if not status: - status = {} - cr = db.cursor() - if cr: - cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'") - if len(cr.fetchall())==0: - logger.notifyChannel("init", netsvc.LOG_INFO, "init db") - tools.init_db(cr) - tools.config["init"]["all"] = 1 - tools.config['update']['all'] = 1 - if not tools.config['without_demo']: - tools.config["demo"]['all'] = 1 - force = [] - if force_demo: - force.append('demo') - - # This is a brand new pool, just created in pooler.get_db_and_pool() - pool = pooler.get_pool(cr.dbname) - - try: - processed_modules = [] - report = tools.assertion_report() - # NOTE: Try to also load the modules that have been marked as uninstallable previously... - STATES_TO_LOAD = ['installed', 'to upgrade', 'uninstallable'] - if 'base' in tools.config['update'] or 'all' in tools.config['update']: - cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) - - # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) - graph = create_graph(cr, ['base'], force) - if not graph: - logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)') - raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)')) - processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)) - - if tools.config['load_language']: - for lang in tools.config['load_language'].split(','): - tools.load_language(cr, lang) - - # STEP 2: Mark other modules to be loaded/updated - if update_module: - modobj = pool.get('ir.module.module') - logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list') - if ('base' in tools.config['init']) or ('base' in tools.config['update']): - modobj.update_list(cr, 1) - - _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) - - mods = [k for k in tools.config['init'] if tools.config['init'][k]] - if mods: - ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) - if ids: - modobj.button_install(cr, 1, ids) - - mods = [k for k in tools.config['update'] if tools.config['update'][k]] - if mods: - ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) - if ids: - modobj.button_upgrade(cr, 1, ids) - - cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) - - STATES_TO_LOAD += ['to install'] - - - # STEP 3: Load marked modules (skipping base which was done in STEP 1) - loop_guardrail = 0 - while True: - loop_guardrail += 1 - if loop_guardrail > 100: - raise ValueError('Possible recursive module tree detected, aborting.') - cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(STATES_TO_LOAD),)) - - module_list = [name for (name,) in cr.fetchall() if name not in graph] - if not module_list: - break - - new_modules_in_graph = upgrade_graph(graph, cr, module_list, force) - if new_modules_in_graph == 0: - # nothing to load - break - - logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) - processed_modules.extend(load_module_graph(cr, graph, status, report=report, skip_modules=processed_modules)) - - # load custom models - cr.execute('select model from ir_model where state=%s', ('manual',)) - for model in cr.dictfetchall(): - pool.get('ir.model').instanciate(cr, 1, model['model'], {}) - - # STEP 4: Finish and cleanup - if processed_modules: - cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") - for (model, name) in cr.fetchall(): - model_obj = pool.get(model) - if model_obj and not isinstance(model_obj, osv.osv.osv_memory): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'object %s (%s) has no access rules!' % (model, name)) - - # Temporary warning while we remove access rights on osv_memory objects, as they have - # been replaced by owner-only access rights - cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""") - for (model, name) in cr.fetchall(): - model_obj = pool.get(model) - if isinstance(model_obj, osv.osv.osv_memory): - logger.notifyChannel('init', netsvc.LOG_WARNING, 'In-memory object %s (%s) should not have explicit access rules!' % (model, name)) - - cr.execute("SELECT model from ir_model") - for (model,) in cr.fetchall(): - obj = pool.get(model) - if obj: - obj._check_removed_columns(cr, log=True) - else: - logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model) - - # Cleanup orphan records - pool.get('ir.model.data')._process_end(cr, 1, processed_modules) - - if report.get_report(): - logger.notifyChannel('init', netsvc.LOG_INFO, report) - - for kind in ('init', 'demo', 'update'): - tools.config[kind] = {} - - cr.commit() - if update_module: - cr.execute("select id,name from ir_module_module where state=%s", ('to remove',)) - for mod_id, mod_name in cr.fetchall(): - cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,)) - for rmod, rid in cr.fetchall(): - uid = 1 - rmod_module= pool.get(rmod) - if rmod_module: - rmod_module.unlink(cr, uid, [rid]) - else: - logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid)) - cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,)) - cr.commit() - # - # TODO: remove menu without actions of children - # - while True: - cr.execute('''delete from - ir_ui_menu - where - (id not IN (select parent_id from ir_ui_menu where parent_id is not null)) - and - (id not IN (select res_id from ir_values where model='ir.ui.menu')) - and - (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''') - cr.commit() - if not cr.rowcount: - break - else: - logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,)) - - cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',)) - cr.commit() - finally: - cr.close() +""" +from openerp.modules import + get_module_resource, + get_module_path # used only by base_module_quality # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/addons/base/ir/ir_ui_menu.py b/openerp/addons/base/ir/ir_ui_menu.py index 99a13b46cff..d67f0c247d1 100644 --- a/openerp/addons/base/ir/ir_ui_menu.py +++ b/openerp/addons/base/ir/ir_ui_menu.py @@ -24,7 +24,7 @@ import base64 import re import tools -import addons +import openerp.modules from osv import fields, osv from tools.translate import _ @@ -229,7 +229,7 @@ class ir_ui_menu(osv.osv): if not path: return False path_info = path.split(',') - icon_path = addons.get_module_resource(path_info[0],path_info[1]) + icon_path = openerp.modules.get_module_resource(path_info[0],path_info[1]) icon_image = False if icon_path: try: diff --git a/openerp/addons/base/module/module.py b/openerp/addons/base/module/module.py index 5458771b49d..36a5f80dfa3 100644 --- a/openerp/addons/base/module/module.py +++ b/openerp/addons/base/module/module.py @@ -30,7 +30,7 @@ import urllib import zipfile import zipimport -import addons +import openerp.modules as addons import pooler import release import tools diff --git a/openerp/db/__init__.py b/openerp/db/__init__.py new file mode 100644 index 00000000000..5eb06af1ad1 --- /dev/null +++ b/openerp/db/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Lower-level database access. + +This module provides access to the underlying database without going +through the ORM. The goal is to gather sql_db.py and other various db +code. + +""" + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/modules/__init__.py b/openerp/modules/__init__.py new file mode 100644 index 00000000000..2c5c3e037fc --- /dev/null +++ b/openerp/modules/__init__.py @@ -0,0 +1,1002 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2009 Tiny SPRL (). +# Copyright (C) 2010-2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Modules (also called addons) management. + +""" + +import os, sys, imp +from os.path import join as opj +import itertools +import zipimport + +import openerp.osv as osv +import openerp.tools as tools +import openerp.tools.osutil as osutil +from openerp.tools.safe_eval import safe_eval as eval +import openerp.pooler as pooler +from openerp.tools.translate import _ + +import openerp.netsvc as netsvc + +import zipfile +import openerp.release as release + +import re +import base64 +from zipfile import PyZipFile, ZIP_DEFLATED +from cStringIO import StringIO + +import logging + +logger = netsvc.Logger() + +_ad = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'addons') # default addons path (base) +ad_paths = [] + +# Modules already loaded +loaded = [] + +def initialize_sys_path(): + global ad_paths + + if ad_paths: + return + + ad_paths = map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(',')) + + sys.path.insert(1, _ad) + + ad_cnt=1 + for adp in ad_paths: + if adp != _ad: + sys.path.insert(ad_cnt, adp) + ad_cnt+=1 + + ad_paths.append(_ad) # for get_module_path + +class Graph(dict): + """ Modules dependency graph. + + The graph is a mapping from module name to Nodes. + + """ + + def addNode(self, name, deps): + max_depth, father = 0, None + for n in [Node(x, self) for x in deps]: + if n.depth >= max_depth: + father = n + max_depth = n.depth + if father: + father.addChild(name) + else: + Node(name, self) + + def update_from_db(self, cr): + if not len(self): + return + # update the graph with values from the database (if exist) + ## First, we set the default values for each package in graph + additional_data = dict.fromkeys(self.keys(), {'id': 0, 'state': 'uninstalled', 'dbdemo': False, 'installed_version': None}) + ## Then we get the values from the database + cr.execute('SELECT name, id, state, demo AS dbdemo, latest_version AS installed_version' + ' FROM ir_module_module' + ' WHERE name IN %s',(tuple(additional_data),) + ) + + ## and we update the default values with values from the database + additional_data.update(dict([(x.pop('name'), x) for x in cr.dictfetchall()])) + + for package in self.values(): + for k, v in additional_data[package.name].items(): + setattr(package, k, v) + + def __iter__(self): + level = 0 + done = set(self.keys()) + while done: + level_modules = [(name, module) for name, module in self.items() if module.depth==level] + for name, module in level_modules: + done.remove(name) + yield module + level += 1 + +class Singleton(object): + def __new__(cls, name, graph): + if name in graph: + inst = graph[name] + else: + inst = object.__new__(cls) + inst.name = name + graph[name] = inst + return inst + + +class Node(Singleton): + """ One module in the modules dependency graph. + + Node acts as a per-module singleton. + + """ + + def __init__(self, name, graph): + self.graph = graph + if not hasattr(self, 'children'): + self.children = [] + if not hasattr(self, 'depth'): + self.depth = 0 + + def addChild(self, name): + node = Node(name, self.graph) + node.depth = self.depth + 1 + if node not in self.children: + self.children.append(node) + for attr in ('init', 'update', 'demo'): + if hasattr(self, attr): + setattr(node, attr, True) + self.children.sort(lambda x, y: cmp(x.name, y.name)) + + def __setattr__(self, name, value): + super(Singleton, self).__setattr__(name, value) + if name in ('init', 'update', 'demo'): + tools.config[name][self.name] = 1 + for child in self.children: + setattr(child, name, value) + if name == 'depth': + for child in self.children: + setattr(child, name, value + 1) + + def __iter__(self): + return itertools.chain(iter(self.children), *map(iter, self.children)) + + def __str__(self): + return self._pprint() + + def _pprint(self, depth=0): + s = '%s\n' % self.name + for c in self.children: + s += '%s`-> %s' % (' ' * depth, c._pprint(depth+1)) + return s + + +def get_module_path(module, downloaded=False): + """Return the path of the given module. + + Search the addons paths and return the first path where the given + module is found. If downloaded is True, return the default addons + path if nothing else is found. + + """ + initialize_sys_path() + for adp in ad_paths: + if os.path.exists(opj(adp, module)) or os.path.exists(opj(adp, '%s.zip' % module)): + return opj(adp, module) + + if downloaded: + return opj(_ad, module) + logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: module not found' % (module,)) + return False + + +def get_module_filetree(module, dir='.'): + path = get_module_path(module) + if not path: + return False + + dir = os.path.normpath(dir) + if dir == '.': + dir = '' + if dir.startswith('..') or (dir and dir[0] == '/'): + raise Exception('Cannot access file outside the module') + + if not os.path.isdir(path): + # zipmodule + zip = zipfile.ZipFile(path + ".zip") + files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] + else: + files = osutil.listdir(path, True) + + tree = {} + for f in files: + if not f.startswith(dir): + continue + + if dir: + f = f[len(dir)+int(not dir.endswith('/')):] + lst = f.split(os.sep) + current = tree + while len(lst) != 1: + current = current.setdefault(lst.pop(0), {}) + current[lst.pop(0)] = None + + return tree + +def zip_directory(directory, b64enc=True, src=True): + """Compress a directory + + @param directory: The directory to compress + @param base64enc: if True the function will encode the zip file with base64 + @param src: Integrate the source files + + @return: a string containing the zip file + """ + + RE_exclude = re.compile('(?:^\..+\.swp$)|(?:\.py[oc]$)|(?:\.bak$)|(?:\.~.~$)', re.I) + + def _zippy(archive, path, src=True): + path = os.path.abspath(path) + base = os.path.basename(path) + for f in osutil.listdir(path, True): + bf = os.path.basename(f) + if not RE_exclude.search(bf) and (src or bf in ('__openerp__.py', '__terp__.py') or not bf.endswith('.py')): + archive.write(os.path.join(path, f), os.path.join(base, f)) + + archname = StringIO() + archive = PyZipFile(archname, "w", ZIP_DEFLATED) + + # for Python 2.5, ZipFile.write() still expects 8-bit strings (2.6 converts to utf-8) + directory = tools.ustr(directory).encode('utf-8') + + archive.writepy(directory) + _zippy(archive, directory, src=src) + archive.close() + archive_data = archname.getvalue() + archname.close() + + if b64enc: + return base64.encodestring(archive_data) + + return archive_data + +def get_module_as_zip(modulename, b64enc=True, src=True): + """Generate a module as zip file with the source or not and can do a base64 encoding + + @param modulename: The module name + @param b64enc: if True the function will encode the zip file with base64 + @param src: Integrate the source files + + @return: a stream to store in a file-like object + """ + + ap = get_module_path(str(modulename)) + if not ap: + raise Exception('Unable to find path for module %s' % modulename) + + ap = ap.encode('utf8') + if os.path.isfile(ap + '.zip'): + val = file(ap + '.zip', 'rb').read() + if b64enc: + val = base64.encodestring(val) + else: + val = zip_directory(ap, b64enc, src) + + return val + + +def get_module_resource(module, *args): + """Return the full path of a resource of the given module. + + @param module: the module + @param args: the resource path components + + @return: absolute path to the resource + + TODO name it get_resource_path + TODO make it available inside on osv object (self.get_resource_path) + """ + a = get_module_path(module) + if not a: return False + resource_path = opj(a, *args) + if zipfile.is_zipfile( a +'.zip') : + zip = zipfile.ZipFile( a + ".zip") + files = ['/'.join(f.split('/')[1:]) for f in zip.namelist()] + resource_path = '/'.join(args) + if resource_path in files: + return opj(a, resource_path) + elif os.path.exists(resource_path): + return resource_path + return False + +def get_modules(): + """Returns the list of module names + """ + def listdir(dir): + def clean(name): + name = os.path.basename(name) + if name[-4:] == '.zip': + name = name[:-4] + return name + + def is_really_module(name): + name = opj(dir, name) + return os.path.isdir(name) or zipfile.is_zipfile(name) + return map(clean, filter(is_really_module, os.listdir(dir))) + + plist = [] + initialize_sys_path() + for ad in ad_paths: + plist.extend(listdir(ad)) + return list(set(plist)) + +def load_information_from_description_file(module): + """ + :param module: The name of the module (sale, purchase, ...) + """ + + for filename in ['__openerp__.py', '__terp__.py']: + description_file = get_module_resource(module, filename) + if description_file : + desc_f = tools.file_open(description_file) + try: + return eval(desc_f.read()) + finally: + desc_f.close() + + #TODO: refactor the logger in this file to follow the logging guidelines + # for 6.0 + logging.getLogger('addons').debug('The module %s does not contain a description file:'\ + '__openerp__.py or __terp__.py (deprecated)', module) + return {} + +def get_modules_with_version(): + modules = get_modules() + res = {} + for module in modules: + try: + info = load_information_from_description_file(module) + res[module] = "%s.%s" % (release.major_version, info['version']) + except Exception, e: + continue + return res + +def create_graph(cr, module_list, force=None): + graph = Graph() + upgrade_graph(graph, cr, module_list, force) + return graph + +def upgrade_graph(graph, cr, module_list, force=None): + if force is None: + force = [] + packages = [] + len_graph = len(graph) + for module in module_list: + mod_path = get_module_path(module) + terp_file = get_module_resource(module, '__openerp__.py') + if not terp_file: + terp_file = get_module_resource(module, '__terp__.py') + if not mod_path or not terp_file: + logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not found, skipped' % (module)) + continue + + if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path+'.zip'): + terp_f = tools.file_open(terp_file) + try: + info = eval(terp_f.read()) + except Exception: + logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: eval file %s' % (module, terp_file)) + raise + finally: + terp_f.close() + if info.get('installable', True): + packages.append((module, info.get('depends', []), info)) + else: + logger.notifyChannel('init', netsvc.LOG_WARNING, 'module %s: not installable, skipped' % (module)) + + dependencies = dict([(p, deps) for p, deps, data in packages]) + current, later = set([p for p, dep, data in packages]), set() + + while packages and current > later: + package, deps, data = packages[0] + + # if all dependencies of 'package' are already in the graph, add 'package' in the graph + if reduce(lambda x, y: x and y in graph, deps, True): + if not package in current: + packages.pop(0) + continue + later.clear() + current.remove(package) + graph.addNode(package, deps) + node = Node(package, graph) + node.data = data + for kind in ('init', 'demo', 'update'): + if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force: + setattr(node, kind, True) + else: + later.add(package) + packages.append((package, deps, data)) + packages.pop(0) + + graph.update_from_db(cr) + + for package in later: + unmet_deps = filter(lambda p: p not in graph, dependencies[package]) + logger.notifyChannel('init', netsvc.LOG_ERROR, 'module %s: Unmet dependencies: %s' % (package, ', '.join(unmet_deps))) + + result = len(graph) - len_graph + if result != len(module_list): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'Not all modules have loaded.') + return result + + +def init_module_objects(cr, module_name, obj_list): + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: creating or updating database tables' % module_name) + todo = [] + for obj in obj_list: + try: + result = obj._auto_init(cr, {'module': module_name}) + except Exception, e: + raise + if result: + todo += result + if hasattr(obj, 'init'): + obj.init(cr) + cr.commit() + todo.sort() + for t in todo: + t[1](cr, *t[2]) + cr.commit() + + +def register_class(m): + """ + Register module named m, if not already registered + """ + + def log(e): + mt = isinstance(e, zipimport.ZipImportError) and 'zip ' or '' + msg = "Couldn't load %smodule %s" % (mt, m) + logger.notifyChannel('init', netsvc.LOG_CRITICAL, msg) + logger.notifyChannel('init', netsvc.LOG_CRITICAL, e) + + global loaded + if m in loaded: + return + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: registering objects' % m) + mod_path = get_module_path(m) + + initialize_sys_path() + try: + zip_mod_path = mod_path + '.zip' + if not os.path.isfile(zip_mod_path): + fm = imp.find_module(m, ad_paths) + try: + imp.load_module(m, *fm) + finally: + if fm[0]: + fm[0].close() + else: + zimp = zipimport.zipimporter(zip_mod_path) + zimp.load_module(m) + except Exception, e: + log(e) + raise + else: + loaded.append(m) + + +class MigrationManager(object): + """ + This class manage the migration of modules + Migrations files must be python files containing a "migrate(cr, installed_version)" function. + Theses files must respect a directory tree structure: A 'migrations' folder which containt a + folder by version. Version can be 'module' version or 'server.module' version (in this case, + the files will only be processed by this version of the server). Python file names must start + by 'pre' or 'post' and will be executed, respectively, before and after the module initialisation + Example: + + + `-- migrations + |-- 1.0 + | |-- pre-update_table_x.py + | |-- pre-update_table_y.py + | |-- post-clean-data.py + | `-- README.txt # not processed + |-- 5.0.1.1 # files in this folder will be executed only on a 5.0 server + | |-- pre-delete_table_z.py + | `-- post-clean-data.py + `-- foo.py # not processed + + This similar structure is generated by the maintenance module with the migrations files get by + the maintenance contract + + """ + def __init__(self, cr, graph): + self.cr = cr + self.graph = graph + self.migrations = {} + self._get_files() + + def _get_files(self): + + """ + import addons.base.maintenance.utils as maintenance_utils + maintenance_utils.update_migrations_files(self.cr) + #""" + + for pkg in self.graph: + self.migrations[pkg.name] = {} + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + continue + + self.migrations[pkg.name]['module'] = get_module_filetree(pkg.name, 'migrations') or {} + self.migrations[pkg.name]['maintenance'] = get_module_filetree('base', 'maintenance/migrations/' + pkg.name) or {} + + def migrate_module(self, pkg, stage): + assert stage in ('pre', 'post') + stageformat = {'pre': '[>%s]', + 'post': '[%s>]', + } + + if not (hasattr(pkg, 'update') or pkg.state == 'to upgrade'): + return + + def convert_version(version): + if version.startswith(release.major_version) and version != release.major_version: + return version # the version number already containt the server version + return "%s.%s" % (release.major_version, version) + + def _get_migration_versions(pkg): + def __get_dir(tree): + return [d for d in tree if tree[d] is not None] + + versions = list(set( + __get_dir(self.migrations[pkg.name]['module']) + + __get_dir(self.migrations[pkg.name]['maintenance']) + )) + versions.sort(key=lambda k: parse_version(convert_version(k))) + return versions + + def _get_migration_files(pkg, version, stage): + """ return a list of tuple (module, file) + """ + m = self.migrations[pkg.name] + lst = [] + + mapping = {'module': opj(pkg.name, 'migrations'), + 'maintenance': opj('base', 'maintenance', 'migrations', pkg.name), + } + + for x in mapping.keys(): + if version in m[x]: + for f in m[x][version]: + if m[x][version][f] is not None: + continue + if not f.startswith(stage + '-'): + continue + lst.append(opj(mapping[x], version, f)) + lst.sort() + return lst + + def mergedict(a, b): + a = a.copy() + a.update(b) + return a + + from openerp.tools.parse_version import parse_version + + parsed_installed_version = parse_version(pkg.installed_version or '') + current_version = parse_version(convert_version(pkg.data.get('version', '0'))) + + versions = _get_migration_versions(pkg) + + for version in versions: + if parsed_installed_version < parse_version(convert_version(version)) <= current_version: + + strfmt = {'addon': pkg.name, + 'stage': stage, + 'version': stageformat[stage] % version, + } + + for pyfile in _get_migration_files(pkg, version, stage): + name, ext = os.path.splitext(os.path.basename(pyfile)) + if ext.lower() != '.py': + continue + mod = fp = fp2 = None + try: + fp = tools.file_open(pyfile) + + # imp.load_source need a real file object, so we create + # one from the file-like object we get from file_open + fp2 = os.tmpfile() + fp2.write(fp.read()) + fp2.seek(0) + try: + mod = imp.load_source(name, pyfile, fp2) + logger.notifyChannel('migration', netsvc.LOG_INFO, 'module %(addon)s: Running migration %(version)s %(name)s' % mergedict({'name': mod.__name__}, strfmt)) + mod.migrate(self.cr, pkg.installed_version) + except ImportError: + logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Unable to load %(stage)s-migration file %(file)s' % mergedict({'file': pyfile}, strfmt)) + raise + except AttributeError: + logger.notifyChannel('migration', netsvc.LOG_ERROR, 'module %(addon)s: Each %(stage)s-migration file must have a "migrate(cr, installed_version)" function' % strfmt) + except: + raise + finally: + if fp: + fp.close() + if fp2: + fp2.close() + if mod: + del mod + +log = logging.getLogger('init') + +def load_module_graph(cr, graph, status=None, perform_checks=True, skip_modules=None, **kwargs): + """Migrates+Updates or Installs all module nodes from ``graph`` + :param graph: graph of module nodes to load + :param status: status dictionary for keeping track of progress + :param perform_checks: whether module descriptors should be checked for validity (prints warnings + for same cases, and even raise osv_except if certificate is invalid) + :param skip_modules: optional list of module names (packages) which have previously been loaded and can be skipped + :return: list of modules that were installed or updated + """ + def process_sql_file(cr, fp): + queries = fp.read().split(';') + for query in queries: + new_query = ' '.join(query.split()) + if new_query: + cr.execute(new_query) + + def load_init_update_xml(cr, m, idref, mode, kind): + for filename in package.data.get('%s_xml' % kind, []): + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, filename)) + _, ext = os.path.splitext(filename) + fp = tools.file_open(opj(m, filename)) + try: + if ext == '.csv': + noupdate = (kind == 'init') + tools.convert_csv_import(cr, m, os.path.basename(filename), fp.read(), idref, mode=mode, noupdate=noupdate) + elif ext == '.sql': + process_sql_file(cr, fp) + elif ext == '.yml': + tools.convert_yaml_import(cr, m, fp, idref, mode=mode, **kwargs) + else: + tools.convert_xml_import(cr, m, fp, idref, mode=mode, **kwargs) + finally: + fp.close() + + def load_demo_xml(cr, m, idref, mode): + for xml in package.data.get('demo_xml', []): + name, ext = os.path.splitext(xml) + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading %s' % (m, xml)) + fp = tools.file_open(opj(m, xml)) + try: + if ext == '.csv': + tools.convert_csv_import(cr, m, os.path.basename(xml), fp.read(), idref, mode=mode, noupdate=True) + elif ext == '.yml': + tools.convert_yaml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) + else: + tools.convert_xml_import(cr, m, fp, idref, mode=mode, noupdate=True, **kwargs) + finally: + fp.close() + + def load_data(cr, module_name, id_map, mode): + _load_data(cr, module_name, id_map, mode, 'data') + + def load_demo(cr, module_name, id_map, mode): + _load_data(cr, module_name, id_map, mode, 'demo') + + def load_test(cr, module_name, id_map, mode): + cr.commit() + if not tools.config.options['test_disable']: + try: + _load_data(cr, module_name, id_map, mode, 'test') + except Exception, e: + logging.getLogger('test').exception('Tests failed to execute in module %s', module_name) + finally: + if tools.config.options['test_commit']: + cr.commit() + else: + cr.rollback() + + def _load_data(cr, module_name, id_map, mode, kind): + for filename in package.data.get(kind, []): + noupdate = (kind == 'demo') + _, ext = os.path.splitext(filename) + log.info("module %s: loading %s", module_name, filename) + pathname = os.path.join(module_name, filename) + file = tools.file_open(pathname) + try: + if ext == '.sql': + process_sql_file(cr, file) + elif ext == '.csv': + noupdate = (kind == 'init') + tools.convert_csv_import(cr, module_name, pathname, file.read(), id_map, mode, noupdate) + elif ext == '.yml': + tools.convert_yaml_import(cr, module_name, file, id_map, mode, noupdate) + else: + tools.convert_xml_import(cr, module_name, file, id_map, mode, noupdate) + finally: + file.close() + + # **kwargs is passed directly to convert_xml_import + if not status: + status = {} + + status = status.copy() + processed_modules = [] + statusi = 0 + pool = pooler.get_pool(cr.dbname) + migrations = MigrationManager(cr, graph) + modobj = None + logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph)) + + for package in graph: + if skip_modules and package.name in skip_modules: + continue + logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name) + migrations.migrate_module(package, 'pre') + register_class(package.name) + modules = pool.instanciate(package.name, cr) + if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): + init_module_objects(cr, package.name, modules) + cr.commit() + + for package in graph: + status['progress'] = (float(statusi)+0.1) / len(graph) + m = package.name + mid = package.id + + if skip_modules and m in skip_modules: + continue + + if modobj is None: + modobj = pool.get('ir.module.module') + + if modobj and perform_checks: + modobj.check(cr, 1, [mid]) + + idref = {} + status['progress'] = (float(statusi)+0.4) / len(graph) + + mode = 'update' + if hasattr(package, 'init') or package.state == 'to install': + mode = 'init' + + if hasattr(package, 'init') or hasattr(package, 'update') or package.state in ('to install', 'to upgrade'): + for kind in ('init', 'update'): + if package.state=='to upgrade': + # upgrading the module information + modobj.write(cr, 1, [mid], modobj.get_values_from_terp(package.data)) + load_init_update_xml(cr, m, idref, mode, kind) + load_data(cr, m, idref, mode) + if hasattr(package, 'demo') or (package.dbdemo and package.state != 'installed'): + status['progress'] = (float(statusi)+0.75) / len(graph) + load_demo_xml(cr, m, idref, mode) + load_demo(cr, m, idref, mode) + cr.execute('update ir_module_module set demo=%s where id=%s', (True, mid)) + + # launch tests only in demo mode, as most tests will depend + # on demo data. Other tests can be added into the regular + # 'data' section, but should probably not alter the data, + # as there is no rollback. + load_test(cr, m, idref, mode) + + processed_modules.append(package.name) + + migrations.migrate_module(package, 'post') + + if modobj: + ver = release.major_version + '.' + package.data.get('version', '1.0') + # Set new modules and dependencies + modobj.write(cr, 1, [mid], {'state': 'installed', 'latest_version': ver}) + cr.commit() + # Update translations for all installed languages + modobj.update_translations(cr, 1, [mid], None, {'overwrite': tools.config['overwrite_existing_translations']}) + cr.commit() + + package.state = 'installed' + for kind in ('init', 'demo', 'update'): + if hasattr(package, kind): + delattr(package, kind) + + statusi += 1 + + cr.commit() + + return processed_modules + +def _check_module_names(cr, module_names): + mod_names = set(module_names) + if 'base' in mod_names: + # ignore dummy 'all' module + if 'all' in mod_names: + mod_names.remove('all') + if mod_names: + cr.execute("SELECT count(id) AS count FROM ir_module_module WHERE name in %s", (tuple(mod_names),)) + if cr.dictfetchone()['count'] != len(mod_names): + # find out what module name(s) are incorrect: + cr.execute("SELECT name FROM ir_module_module") + incorrect_names = mod_names.difference([x['name'] for x in cr.dictfetchall()]) + logging.getLogger('init').warning('invalid module names, ignored: %s', ", ".join(incorrect_names)) + +def load_modules(db, force_demo=False, status=None, update_module=False): + + initialize_sys_path() + + # Backward compatibility: addons don't have to import openerp.xxx, they still can import xxx + for k, v in list(sys.modules.items()): + if k.startswith('openerp.') and sys.modules.get(k[8:]) is None: + sys.modules[k[8:]] = v + + if not status: + status = {} + cr = db.cursor() + if cr: + cr.execute("SELECT relname FROM pg_class WHERE relkind='r' AND relname='ir_module_module'") + if len(cr.fetchall())==0: + logger.notifyChannel("init", netsvc.LOG_INFO, "init db") + tools.init_db(cr) + tools.config["init"]["all"] = 1 + tools.config['update']['all'] = 1 + if not tools.config['without_demo']: + tools.config["demo"]['all'] = 1 + force = [] + if force_demo: + force.append('demo') + + # This is a brand new pool, just created in pooler.get_db_and_pool() + pool = pooler.get_pool(cr.dbname) + + try: + processed_modules = [] + report = tools.assertion_report() + # NOTE: Try to also load the modules that have been marked as uninstallable previously... + STATES_TO_LOAD = ['installed', 'to upgrade', 'uninstallable'] + if 'base' in tools.config['update'] or 'all' in tools.config['update']: + cr.execute("update ir_module_module set state=%s where name=%s and state=%s", ('to upgrade', 'base', 'installed')) + + # STEP 1: LOAD BASE (must be done before module dependencies can be computed for later steps) + graph = create_graph(cr, ['base'], force) + if not graph: + logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)') + raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)')) + processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), report=report)) + + if tools.config['load_language']: + for lang in tools.config['load_language'].split(','): + tools.load_language(cr, lang) + + # STEP 2: Mark other modules to be loaded/updated + if update_module: + modobj = pool.get('ir.module.module') + logger.notifyChannel('init', netsvc.LOG_INFO, 'updating modules list') + if ('base' in tools.config['init']) or ('base' in tools.config['update']): + modobj.update_list(cr, 1) + + _check_module_names(cr, itertools.chain(tools.config['init'].keys(), tools.config['update'].keys())) + + mods = [k for k in tools.config['init'] if tools.config['init'][k]] + if mods: + ids = modobj.search(cr, 1, ['&', ('state', '=', 'uninstalled'), ('name', 'in', mods)]) + if ids: + modobj.button_install(cr, 1, ids) + + mods = [k for k in tools.config['update'] if tools.config['update'][k]] + if mods: + ids = modobj.search(cr, 1, ['&', ('state', '=', 'installed'), ('name', 'in', mods)]) + if ids: + modobj.button_upgrade(cr, 1, ids) + + cr.execute("update ir_module_module set state=%s where name=%s", ('installed', 'base')) + + STATES_TO_LOAD += ['to install'] + + + # STEP 3: Load marked modules (skipping base which was done in STEP 1) + loop_guardrail = 0 + while True: + loop_guardrail += 1 + if loop_guardrail > 100: + raise ValueError('Possible recursive module tree detected, aborting.') + cr.execute("SELECT name from ir_module_module WHERE state IN %s" ,(tuple(STATES_TO_LOAD),)) + + module_list = [name for (name,) in cr.fetchall() if name not in graph] + if not module_list: + break + + new_modules_in_graph = upgrade_graph(graph, cr, module_list, force) + if new_modules_in_graph == 0: + # nothing to load + break + + logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list))) + processed_modules.extend(load_module_graph(cr, graph, status, report=report, skip_modules=processed_modules)) + + # load custom models + cr.execute('select model from ir_model where state=%s', ('manual',)) + for model in cr.dictfetchall(): + pool.get('ir.model').instanciate(cr, 1, model['model'], {}) + + # STEP 4: Finish and cleanup + if processed_modules: + cr.execute("""select model,name from ir_model where id NOT IN (select distinct model_id from ir_model_access)""") + for (model, name) in cr.fetchall(): + model_obj = pool.get(model) + if model_obj and not isinstance(model_obj, osv.osv.osv_memory): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'object %s (%s) has no access rules!' % (model, name)) + + # Temporary warning while we remove access rights on osv_memory objects, as they have + # been replaced by owner-only access rights + cr.execute("""select distinct mod.model, mod.name from ir_model_access acc, ir_model mod where acc.model_id = mod.id""") + for (model, name) in cr.fetchall(): + model_obj = pool.get(model) + if isinstance(model_obj, osv.osv.osv_memory): + logger.notifyChannel('init', netsvc.LOG_WARNING, 'In-memory object %s (%s) should not have explicit access rules!' % (model, name)) + + cr.execute("SELECT model from ir_model") + for (model,) in cr.fetchall(): + obj = pool.get(model) + if obj: + obj._check_removed_columns(cr, log=True) + else: + logger.notifyChannel('init', netsvc.LOG_WARNING, "Model %s is referenced but not present in the orm pool!" % model) + + # Cleanup orphan records + pool.get('ir.model.data')._process_end(cr, 1, processed_modules) + + if report.get_report(): + logger.notifyChannel('init', netsvc.LOG_INFO, report) + + for kind in ('init', 'demo', 'update'): + tools.config[kind] = {} + + cr.commit() + if update_module: + cr.execute("select id,name from ir_module_module where state=%s", ('to remove',)) + for mod_id, mod_name in cr.fetchall(): + cr.execute('select model,res_id from ir_model_data where noupdate=%s and module=%s order by id desc', (False, mod_name,)) + for rmod, rid in cr.fetchall(): + uid = 1 + rmod_module= pool.get(rmod) + if rmod_module: + rmod_module.unlink(cr, uid, [rid]) + else: + logger.notifyChannel('init', netsvc.LOG_ERROR, 'Could not locate %s to remove res=%d' % (rmod,rid)) + cr.execute('delete from ir_model_data where noupdate=%s and module=%s', (False, mod_name,)) + cr.commit() + # + # TODO: remove menu without actions of children + # + while True: + cr.execute('''delete from + ir_ui_menu + where + (id not IN (select parent_id from ir_ui_menu where parent_id is not null)) + and + (id not IN (select res_id from ir_values where model='ir.ui.menu')) + and + (id not IN (select res_id from ir_model_data where model='ir.ui.menu'))''') + cr.commit() + if not cr.rowcount: + break + else: + logger.notifyChannel('init', netsvc.LOG_INFO, 'removed %d unused menus' % (cr.rowcount,)) + + cr.execute("update ir_module_module set state=%s where state=%s", ('uninstalled', 'to remove',)) + cr.commit() + finally: + cr.close() + + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/openerp/pooler.py b/openerp/pooler.py index 0de5f832494..b162bba4b8d 100644 --- a/openerp/pooler.py +++ b/openerp/pooler.py @@ -33,7 +33,7 @@ def get_db_and_pool(db_name, force_demo=False, status=None, update_module=False, if db_name in pool_dic: pool = pool_dic[db_name] else: - import openerp.addons as addons + import openerp.modules import openerp.osv.osv as osv_osv pool = osv_osv.osv_pool() @@ -43,7 +43,7 @@ def get_db_and_pool(db_name, force_demo=False, status=None, update_module=False, # an exception is raised. pool_dic[db_name] = pool try: - addons.load_modules(db, force_demo, status, update_module) + openerp.modules.load_modules(db, force_demo, status, update_module) except Exception: del pool_dic[db_name] raise diff --git a/openerp/report/interface.py b/openerp/report/interface.py index 5bb1224a549..178c015ba34 100644 --- a/openerp/report/interface.py +++ b/openerp/report/interface.py @@ -27,7 +27,7 @@ import openerp.netsvc as netsvc import openerp.pooler as pooler import openerp.tools as tools -import openerp.addons as addons +import openerp.modules import print_xml import render import urllib @@ -119,7 +119,7 @@ class report_rml(report_int): pos_xml = i.end() doc = print_xml.document(cr, uid, {}, {}) - tmpl_path = addons.get_module_resource('base', 'report', 'corporate_defaults.xml') + tmpl_path = openerp.modules.get_module_resource('base', 'report', 'corporate_defaults.xml') doc.parse(tmpl_path, [uid], 'res.users', context) corporate_header = doc.xml_get() doc.close() diff --git a/openerp/service/web_services.py b/openerp/service/web_services.py index 95194679198..e19fc523e3c 100644 --- a/openerp/service/web_services.py +++ b/openerp/service/web_services.py @@ -28,7 +28,7 @@ import time import sys import platform from openerp.tools.translate import _ -import openerp.addons as addons +import openerp.modules import openerp.ir import openerp.netsvc as netsvc import openerp.pooler as pooler @@ -139,7 +139,7 @@ class db(netsvc.ExportService): def exp_get_progress(self, id): if self.actions[id]['thread'].isAlive(): -# return addons.init_progress[db_name] +# return openerp.modules.init_progress[db_name] return (min(self.actions[id].get('progress', 0),0.95), []) else: clean = self.actions[id]['clean'] @@ -449,7 +449,7 @@ GNU Public Licence. if not rc.id: raise tm.RemoteContractException('This contract does not exist or is not active') - return rc.get_available_updates(rc.id, addons.get_modules_with_version()) + return rc.get_available_updates(rc.id, openerp.modules.get_modules_with_version()) except tm.RemoteContractException, e: self.abortResponse(1, 'Migration Error', 'warning', str(e)) @@ -467,7 +467,7 @@ GNU Public Licence. l.notifyChannel('migration', netsvc.LOG_INFO, 'starting migration with contract %s' % (rc.name,)) - zips = rc.retrieve_updates(rc.id, addons.get_modules_with_version()) + zips = rc.retrieve_updates(rc.id, openerp.modules.get_modules_with_version()) from shutil import rmtree, copytree, copy @@ -479,7 +479,7 @@ GNU Public Licence. for module in zips: l.notifyChannel('migration', netsvc.LOG_INFO, 'upgrade module %s' % (module,)) - mp = addons.get_module_path(module) + mp = openerp.modules.get_module_path(module) if mp: if os.path.isdir(mp): copytree(mp, os.path.join(backup_directory, module)) diff --git a/openerp/tools/misc.py b/openerp/tools/misc.py index e1a6a717e87..276274563be 100644 --- a/openerp/tools/misc.py +++ b/openerp/tools/misc.py @@ -71,7 +71,7 @@ SKIPPED_ELEMENT_TYPES = (etree._Comment, etree._ProcessingInstruction, etree.Com # initialize a database with base/base.sql def init_db(cr): - import openerp.addons as addons + import openerp.modules as addons f = addons.get_module_resource('base', 'base.sql') base_sql_file = file_open(f) try: @@ -204,7 +204,7 @@ def file_open(name, mode="r", subdir='addons', pathinfo=False): @return: fileobject if pathinfo is False else (fileobject, filepath) """ - import openerp.addons as addons + import openerp.modules as addons adps = addons.ad_paths rtp = os.path.normcase(os.path.abspath(config['root_path']))