[ADD] import framework

bzr revid: tfr@openerp.com-20110512084908-j3o9lsrja52ybflf
This commit is contained in:
tfr@openerp.com 2011-05-12 10:49:08 +02:00
parent 12f8701002
commit 8752ac03b4
4 changed files with 649 additions and 0 deletions

View File

@ -0,0 +1,23 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# Copyright (C) 2004-2010 Openerp sa (<http://openerp.com>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import import_framework
import mapper

View File

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
'name': 'Base framework for module that need to import complex data',
'version': '0.9',
'category': 'Generic Modules',
'description': """
This module provide a class import_framework to help importing
complex data from other software
""",
'author': 'OpenERP SA',
'website': 'http://www.openerp.com',
'depends': ['base'],
'init_xml': [],
'update_xml': [],
'demo_xml': [],
'test': [], #TODO provide test
'installable': True,
'active': False,
}

View File

@ -0,0 +1,420 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import pprint
import mapper
pp = pprint.PrettyPrinter(indent=4)
class import_framework():
"""
This class should be extends,
get_data and get_mapping have to extends
get_state_map and initialize can be extended
for advanced purpose get_default_hook can also be extended
@see dummy import for a minimal exemple
"""
"""
for import_object, this domain will avoid to find an already existing object
"""
DO_NOT_FIND_DOMAIN = [('id', '=', 0)]
def __init__(self, obj, cr, uid, instance_name, module_name, context):
self.obj = obj
self.cr = cr
self.uid = uid
self.instance_name = instance_name
self.module_name = module_name
self.context = context or {}
self.initialize()
"""
Abstract Method to be implemented in
the real instance
"""
def initialize(self):
"""
init before import
usually for the login
"""
pass
def get_data(self, table):
"""
@return: a list of dictionaries
each dictionnaries contains the list of pair external_field_name : value
"""
return [{}]
def get_mapping(self):
"""
@return: { TABLE_NAME : {
'model' : 'openerp.model.name',
#if true import the table if not just resolve dependencies, use for meta package, by default => True
#Not required
'import' : True or False,
#Not required
'dependencies' : [TABLE_1, TABLE_2],
#Not required
'hook' : self.function_name, #get the val dict of the object, return the same val dict or False
'map' : { @see mapper
'openerp_field_name' : 'external_field_name', or val('external_field_name')
'openerp_field_id/id' : ref(TABLE_1, 'external_id_field'), #make the mapping between the external id and the xml on the right
'openerp_field2_id/id_parent' : ref(TABLE_1,'external_id_field') #indicate a self dependencies on openerp_field2_id
'state' : map_val('state_equivalent_field', mapping), # use get_state_map to make the mapping between the value of the field and the value of the state
'text_field' : concat('field_1', 'field_2', .., delimiter=':'), #concat the value of the list of field in one
'description_field' : ppconcat('field_1', 'field_2', .., delimiter='\n\t'), #same as above but with a prettier formatting
'field' : call(callable, arg1, arg2, ..), #call the function with all the value, the function should send the value : self.callable
'field' : callable
'field' : call(method, val('external_field') interface of method is self, val where val is the value of the field
'field' : const(value) #always set this field to value
+ any custom mapper that you will define
}
},
}
"""
return {}
def default_hook(self, val):
"""
this hook will be apply on each table that don't have hook
here we define the identity hook
"""
return val
def _import_table(self, table):
data = self.get_data(table)
map = self.get_mapping()[table]['map']
hook = self.get_mapping()[table].get('hook', self.default_hook)
model = self.get_mapping()[table]['model']
final_data = []
for val in data:
res = hook(val)
if res:
final_data.append(res)
self._save_data(model, map, final_data, table)
def _save_data(self, model, mapping, datas, table):
"""
@param model: the model of the object to import
@param table : the external table where the data come from
@param mapping : definition of the mapping
@see: get_mapping
@param datas : list of dictionnaries
datas = [data_1, data_2, ..]
data_i is a map external field_name => value
and each data_i have a external id => in data_id['id']
"""
if not datas:
return
mapping['id'] = 'id_new'
res = []
for data in datas:
self_dependencies = []
for k in mapping.keys():
if '_parent' in k:
self_dependencies.append(k[:-7])
field_name = mapping.pop(k)
data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table)
data['id_new'] = self._generate_xml_id(data['id'], table)
fields, values = self._fields_mapp(data, mapping, table)
res.append(values)
model_obj = self.obj.pool.get(model)
if not model_obj:
raise ValueError("%s is not a valid model name" % model)
model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context)
for field in self_dependencies:
self._import_self_dependencies(model_obj, field, datas)
def _import_self_dependencies(self, obj, parent_field, datas):
"""
@param parent_field: the name of the field that generate a self_dependencies, we call the object referenced in this
field the parent of the object
@param datas: a list of dictionnaries
Dictionnaries need to contains
id_new : the xml_id of the object
field_new : the xml_id of the parent
"""
fields = ['id', parent_field]
for data in datas:
if data.get(parent_field + '_parent'):
values = [data['id_new'], data[parent_field + '_parent']]
obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=True, context=self.context)
def _preprocess_mapping(self, mapping):
"""
Preprocess the mapping :
after the preprocces, everything is
callable in the val of the dictionary
use to allow syntaxical sugar like 'field': 'external_field'
instead of 'field' : value('external_field')
"""
map = dict(mapping)
for key, value in map.items():
if isinstance(value, basestring):
map[key] = mapper.value(value)
#set parent for instance of dbmapper
elif isinstance(value, mapper.dbmapper):
value.set_parent(self)
return map
def _fields_mapp(self,dict_sugar, openerp_dict, table):
"""
call all the mapper and transform data
to be compatible with import_data
"""
fields=[]
data_lst = []
mapping = self._preprocess_mapping(openerp_dict)
for key,val in mapping.items():
if key not in fields and dict_sugar:
fields.append(key)
value = val(dict(dict_sugar))
data_lst.append(value)
return fields, data_lst
def _generate_xml_id(self, name, table):
"""
@param name: name of the object, has to be unique in for a given table
@param table : table where the record we want generate come from
@return: a unique xml id for record, the xml_id will be the same given the same table and same name
To be used to avoid duplication of data that don't have ids
"""
sugar_instance = "sugarcrm" #TODO need to be changed we information is known in the wizard
name = name.replace('.', '_').replace(',', '_')
return sugar_instance + "_" + table + "_" + name
"""
Public interface of the framework
those function can be use in the callable function defined in the mapping
"""
def xml_id_exist(self, table, external_id):
"""
Check if the external id exist in the openerp database
in order to check if the id exist the table where it come from
should be provide
"""
if not external_id:
return False
xml_id = self._generate_xml_id(external_id, table)
id = self.obj.pool.get('ir.model.data').search(self.cr, self.uid, [('name', '=', xml_id), ('module', '=', self.module_name)])
return id and xml_id or False
def name_exist(self, table, name, model):
"""
Check if the object with the name exist in the openerp database
in order to check if the id exist the table where it come from
should be provide and the model of the object
"""
fields = ['name']
data = [name]
return self.import_object(fields, data, model, table, name, [('name', '=', name)])
def get_mapped_id(self, table, external_id, context=None):
"""
@return return the databse id linked with the external_id
"""
if not external_id:
return False
xml_id = self._generate_xml_id(external_id, table)
return self.obj.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, self.module_name, xml_id)[1]
def import_object_mapping(self, mapping, data, model, table, name, domain_search=False):
"""
same as import_objects but instead of two list fields and data,
this method take a dictionnaries : external_field : value
and the mapping similar to the one define in 'map' key
@see import_object, get_mapping
"""
fields, datas = self._fields_mapp(data, mapping, table)
return self.import_object(fields, datas, model, table, name, domain_search)
def import_object(self, fields, data, model, table, name, domain_search=False):
"""
This method will import an object in the openerp, usefull for field that is only a char in sugar and is an object in openerp
use import_data that will take care to create/update or do nothing with the data
this method return the xml_id
To be use, when you want to create an object or link if already exist
use DO_NOT_LINK_DOMAIN to create always a new object
@param fields: list of fields needed to create the object without id
@param data: the list of the data, in the same order as the field
ex : fields = ['firstname', 'lastname'] ; data = ['John', 'Mc donalds']
@param model: the openerp's model of the create/update object
@param table: the table where data come from in sugarcrm, no need to fit the real name of openerp name, just need to be unique
@param unique_name: the name of the object that we want to create/update get the id
@param domain_search : the domain that should find the unique existing record
@return: the xml_id of the ressources
"""
domain_search = not domain_search and [('name', 'ilike', name)] or domain_search
obj = self.obj.pool.get(model)
xml_id = self._generate_xml_id(name, table)
xml_ref = self.mapped_id_if_exist(model, domain_search, table, name)
fields.append('id')
data.append(xml_id)
obj.import_data(self.cr, self.uid, fields, [data], mode='update', current_module=self.module_name, noupdate=True, context=self.context)
return xml_ref or xml_id
def mapped_id_if_exist(self, model, domain, table, name):
"""
To be use when we want link with and existing object, if the object don't exist
just ignore.
@param domain : search domain to find existing record, should return a unique record
@param xml_id: xml_id give to the mapping
@param name: external_id or name of the object to create if there is no id
@param table: the name of the table of the object to map
@return : the xml_id if the record exist in the db, False otherwise
"""
obj = self.obj.pool.get(model)
ids = obj.search(self.cr, self.uid, domain, context=self.context)
if ids:
xml_id = self._generate_xml_id(name, table)
ir_model_data_obj = obj.pool.get('ir.model.data')
id = ir_model_data_obj._update(self.cr, self.uid, model,
self.module_name, {}, mode='update', xml_id=xml_id,
noupdate=True, res_id=ids[0], context=self.context)
return xml_id
return False
def import_all(self, table_list):
"""Import all data into openerp,
this is the Entry point to launch the process of import
@param table_list: the list of external table to import
['Leads', 'Opportunity']
"""
imported = set() #to invoid importing 2 times the sames modules
for table in table_list:
to_import = self.get_mapping()[table].get('import', True)
if not table in imported:
self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported)
if to_import:
self._import_table(table)
imported.add(table)
def _resolve_dependencies(self, dep, imported):
"""
import dependencies recursively
and avoid to import twice the same table
"""
for dependency in dep:
if not dependency in imported:
to_import = self.get_mapping()[dependency].get('import', True)
self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported)
if to_import:
self._import_table(dependency)
imported.add(dependency)
class dummy_import(import_framework):
"""
this is a really simple exemple to show how to use the framework
"""
TABLE_LEAD = 'Leads'
TABLE_STAGE = 'Stage'
TABLE_USER = 'User'
def get_lead_status(self, sugar_val):
fields = ['name', 'type']
name = 'lead_' + sugar_val.get('status', '')
data = [sugar_val.get('status', ''), 'lead']
return self.import_object(fields, data, 'crm.case.stage', self.TABLE_STAGE, name, [('type', '=', 'lead'), ('name', 'ilike', sugar_val.get('status', ''))])
lead_state = {
'New' : 'draft',
'Assigned':'open',
'In Progress': 'open',
'Recycled': 'cancel',
'Dead': 'done',
}
def get_mapping(self):
return {
self.TABLE_LEAD : {
'model' : 'crm.lead',
'dependencies' : [self.TABLE_USER],
'map' : {
'name' : 'name',
'stage_id/id' : self.get_lead_status,
'state' : mapper.map_val('status', self.lead_state),
'user_id/id' : mapper.ref(self.TABLE_USER, 'assigned_user_id'),
}
},
self.TABLE_USER : {
'model' : 'res.users',
'map' : {
'name' : 'name',
'login' : 'name'
}
}
}
#DUMMY way to get data, just demonstration
def get_data(self, table):
if table == self.TABLE_LEAD:
return self.get_lead()
if table == self.TABLE_USER:
return self.get_user()
def get_lead(self):
return [
{'id' : 'lead1',
'name' : 'name1',
'status' : 'Assigned',
'assigned_user_id' : 'user1'},
{'id' : 'lead2',
'name' : 'name2'
},
]
def get_user(self):
return [{
'id' : 'user1',
'name' : 'name1',
}]

View File

@ -0,0 +1,167 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# OpenERP, Open Source Management Solution
# Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
class mapper(object):
"""
super class for all mapper class
They are call before import data
to transform the mapping into real value that we
will import
the call function receive a dictionary with external data
'external_field' : value
"""
def __call__(self, external_values):
raise NotImplementedError()
class dbmapper(mapper):
"""
Super class for mapper that need to access to
data base or any function of the import_framework
self.parent contains a reference to the instance of
the import framework
"""
def set_parent(self, parent):
self.parent = parent
class concat(mapper):
"""
Use : contact('field_name1', 'field_name2', delimiter='_')
concat value of fields using the delimiter, delimiter is optional
and by default is a space
"""
def __init__(self, *arg, **delimiter):
self.arg = arg
self.delimiter = delimiter and delimiter.get('delimiter', ' ') or ' '
def __call__(self, external_values):
return self.delimiter.join(map(lambda x : external_values.get(x,''), self.arg))
class ppconcat(mapper):
"""
Use : contact('field_name1', 'field_name2', delimiter='_')
concat external field name and value of fields using the delimiter,
delimiter is optional and by default is a two line feeds
"""
def __init__(self, *arg, **delimiter):
self.arg = arg
self.delimiter = delimiter and delimiter.get('delimiter', ' ') or '\n\n'
def __call__(self, external_values):
return self.delimiter.join(map(lambda x : x + ": " + external_values.get(x,''), self.arg))
class const(mapper):
"""
Use : const(arg)
return always arg
"""
def __init__(self, val):
self.val = val
def __call__(self, external_values):
return self.val
class value(mapper):
"""
Use : value(external_field_name)
Return the value of the external field name
this is equivalent to the a single string
usefull for call if you want your call get the value
and don't care about the name of the field
call(self.method, value('field1'))
"""
def __init__(self, val, default=''):
self.val = val
self.default = default
def __call__(self, external_values):
return external_values.get(self.val, self.default)
class map_val(mapper):
"""
Use : map_val(external_field, val_mapping)
where val_mapping is a dictionary
with external_val : openerp_val
usefull for selection field like state
to map value
"""
def __init__(self, val, map, default='draft'):
self.val = value(val)
self.map = map
self.default = default
def __call__(self, external_values):
return self.map.get(self.val(external_values), self.default)
class ref(dbmapper):
"""
Use : ref(table_name, external_id)
return the xml_id of the ressource
to associate an already imported object with the current object
"""
def __init__(self, table, field_name):
self.table = table
self.field_name = field_name
def __call__(self, external_values):
return self.parent.xml_id_exist(self.table, external_values.get(self.field_name))
class refbyname(dbmapper):
"""
Use : refbyname(table_name, external_name, res.model)
same as ref but use the name of the ressource to find it
"""
def __init__(self, table, field_name, model):
self.table = table
self.field_name = field_name
self.model = model
def __call__(self, external_values):
v = external_values.get(self.field_name, '')
return self.parent.name_exist(self.table, v , self.model)
class call(mapper):
"""
Use : call(function, arg1, arg2)
to call the function with external val follow by the arg specified
"""
def __init__(self, fun, *arg):
self.fun = fun
self.arg = arg
def __call__(self, external_values):
args = []
for arg in self.arg:
if isinstance(arg, mapper):
args.append(arg(external_values))
else:
args.append(arg)
return self.fun(external_values, *args)