diff --git a/addons/base_import/__init__.py b/addons/base_import/__init__.py new file mode 100644 index 00000000000..f0736de3bd1 --- /dev/null +++ b/addons/base_import/__init__.py @@ -0,0 +1,2 @@ +import models +import tests.models diff --git a/addons/base_import/__openerp__.py b/addons/base_import/__openerp__.py new file mode 100644 index 00000000000..7f117349ba9 --- /dev/null +++ b/addons/base_import/__openerp__.py @@ -0,0 +1,30 @@ +{ + 'name': 'Base import', + 'description': """ +New extensible file import for OpenERP +====================================== + +Re-implement openerp's file import system: + +* Server side, the previous system forces most of the logic into the + client which duplicates the effort (between clients), makes the + import system much harder to use without a client (direct RPC or + other forms of automation) and makes knowledge about the + import/export system much harder to gather as it is spread over + 3+ different projects. + +* In a more extensible manner, so users and partners can build their + own front-end to import from other file formats (e.g. OpenDocument + files) which may be simpler to handle in their work flow or from + their data production sources. + +* In a module, so that administrators and users of OpenERP who do not + need or want an online import can avoid it being available to users. +""", + 'category': 'Uncategorized', + 'website': 'http://www.openerp.com', + 'author': 'OpenERP SA', + 'depends': ['base'], + 'installable': True, + 'auto_install': False, # set to true and allow uninstall? +} diff --git a/addons/base_import/models.py b/addons/base_import/models.py new file mode 100644 index 00000000000..a5e2678cd73 --- /dev/null +++ b/addons/base_import/models.py @@ -0,0 +1,127 @@ +import csv +import itertools + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +from openerp.osv import orm, fields +from openerp.tools.translate import _ + +FIELDS_RECURSION_LIMIT = 2 +class ir_import(orm.TransientModel): + _name = 'base_import.import' + + _columns = { + 'res_model': fields.char('Model', size=64), + 'file': fields.binary('File'), + 'file_name': fields.char('File Name', size=None), + } + + def get_fields(self, cr, uid, model, context=None, + depth=FIELDS_RECURSION_LIMIT): + """ Recursively get fields for the provided model (through + fields_get) and filter them according to importability + + The output format is a list of ``Field``, with ``Field`` + defined as: + + .. class:: Field + + .. attribute:: id (str) + + A non-unique identifier for the field, used to compute + the span of the ``required`` attribute: if multiple + ``required`` fields have the same id, only one of them + is necessary. + + .. attribute:: name (str) + + The field's logical (OpenERP) name within the scope of + its parent. + + .. attribute:: string (str) + + The field's human-readable name (``@string``) + + .. attribute:: required (bool) + + Whether the field is marked as required in the + model. Clients must provide non-empty import values + for all required fields or the import will error out. + + .. attribute:: fields (list(Field)) + + The current field's subfields. The database and + external identifiers for m2o and m2m fields; a + filtered and transformed fields_get for o2m fields (to + a variable depth defined by ``depth``). + + Fields with no sub-fields will have an empty list of + sub-fields. + + :param str model: name of the model to get fields form + :param int landing: depth of recursion into o2m fields + """ + fields = [{ + 'id': 'id', + 'name': 'id', + 'string': _("External ID"), + 'required': False, + 'fields': [], + }] + fields_got = self.pool[model].fields_get(cr, uid, context=context) + for name, field in fields_got.iteritems(): + if field.get('readonly'): + states = field.get('states') + if not states: + continue + # states = {state: [(attr, value), (attr2, value2)], state2:...} + if not any(attr == 'readonly' and value is False + for attr, value in itertools.chain.from_iterable( + states.itervalues())): + continue + + f = { + 'id': name, + 'name': name, + 'string': field['string'], + # Y U NO ALWAYS HAVE REQUIRED + 'required': bool(field.get('required')), + 'fields': [], + } + + if field['type'] in ('many2many', 'many2one'): + f['fields'] = [ + dict(f, name='id', string=_("External ID")), + dict(f, name='.id', string=_("Database ID")), + ] + elif field['type'] == 'one2many' and depth: + f['fields'] = self.get_fields( + cr, uid, field['relation'], context=context, depth=depth-1) + + fields.append(f) + + return fields + + def parse_preview(self, cr, uid, id, options, count=10, context=None): + """ Generates a preview of the uploaded files, and performs + fields-matching between the import's file data and the model's + columns. + + :param id: identifier of the import + :param int count: number of preview lines to generate + :param dict options: format-specific options + :returns: (fields, matches, preview) + :rtype: (dict(str: dict(...)), dict(str, str), list(list(str))) + """ + record = self.browse(cr, uid, id, context=context) + # recursive fields_get (cache based on res_model?) + fields = record.get_fields(record.res_model) + import pprint + pprint.pprint(fields) + # extract title row? + # match title row to fields_ge + # Extract first $count rows + # return triplet diff --git a/addons/base_import/tests/__init__.py b/addons/base_import/tests/__init__.py new file mode 100644 index 00000000000..abef28c8ca1 --- /dev/null +++ b/addons/base_import/tests/__init__.py @@ -0,0 +1,3 @@ +from . import test_cases + +checks = [test_cases] diff --git a/addons/base_import/tests/models.py b/addons/base_import/tests/models.py new file mode 100644 index 00000000000..2eeea1543d8 --- /dev/null +++ b/addons/base_import/tests/models.py @@ -0,0 +1,92 @@ +from openerp.osv import orm, fields + +def name(n): return 'base_import.tests.models.%s' % n + +class char(orm.Model): + _name = name('char') + + _columns = { + 'value': fields.char('unknown', size=None) + } + +class char_required(orm.Model): + _name = name('char.required') + + _columns = { + 'value': fields.char('unknown', size=None, required=True) + } + +class char_readonly(orm.Model): + _name = name('char.readonly') + + _columns = { + 'value': fields.char('unknown', size=None, readonly=True) + } + +class char_states(orm.Model): + _name = name('char.states') + + _columns = { + 'value': fields.char('unknown', size=None, readonly=True, states={'draft': [('readonly', False)]}) + } + +class char_noreadonly(orm.Model): + _name = name('char.noreadonly') + + _columns = { + 'value': fields.char('unknown', size=None, readonly=True, states={'draft': [('invisible', True)]}) + } + +class char_stillreadonly(orm.Model): + _name = name('char.stillreadonly') + + _columns = { + 'value': fields.char('unknown', size=None, readonly=True, states={'draft': [('readonly', True)]}) + } + +# TODO: complex field (m2m, o2m, m2o) +class m2o(orm.Model): + _name = name('m2o') + + _columns = { + 'value': fields.many2one(name('m2o.related')) + } +class m2o_related(orm.Model): + _name = name('m2o.related') + + _columns = { + 'value': fields.integer() + } + _defaults = { + 'value': 42 + } + +class m2o_required(orm.Model): + _name = name('m2o.required') + + _columns = { + 'value': fields.many2one(name('m2o.required.related'), required=True) + } +class m2o_required_related(orm.Model): + _name = name('m2o.required.related') + + _columns = { + 'value': fields.integer() + } + _defaults = { + 'value': 42 + } + +class o2m(orm.Model): + _name = name('o2m') + + _columns = { + 'value': fields.one2many(name('o2m.child'), 'parent_id') + } +class o2m_child(orm.Model): + _name = name('o2m.child') + + _columns = { + 'parent_id': fields.many2one(name('o2m')), + 'value': fields.integer() + } diff --git a/addons/base_import/tests/test_cases.py b/addons/base_import/tests/test_cases.py new file mode 100644 index 00000000000..d415c348f30 --- /dev/null +++ b/addons/base_import/tests/test_cases.py @@ -0,0 +1,73 @@ +from openerp.tests.common import TransactionCase + +ID_FIELD = {'id': 'id', 'name': 'id', 'string': "External ID", 'required': False, 'fields': []} +def make_field(name='value', string='unknown', required=False, fields=[]): + return [ + ID_FIELD, + {'id': name, 'name': name, 'string': string, 'required': required, 'fields': fields}, + ] + +class test_basic_fields(TransactionCase): + def get_fields(self, field): + return self.registry('base_import.import')\ + .get_fields(self.cr, self.uid, 'base_import.tests.models.' + field) + + def test_base(self): + """ A basic field is not required """ + self.assertEqual(self.get_fields('char'), make_field()) + + def test_required(self): + """ Required fields should be flagged (so they can be fill-required) """ + self.assertEqual(self.get_fields('char.required'), make_field(required=True)) + + def test_readonly(self): + """ Readonly fields should be filtered out""" + self.assertEqual(self.get_fields('char.readonly'), [ID_FIELD]) + + def test_readonly_states(self): + """ Readonly fields with states should not be filtered out""" + self.assertEqual(self.get_fields('char.states'), make_field()) + + def test_readonly_states_noreadonly(self): + """ Readonly fields with states having nothing to do with + readonly should still be filtered out""" + self.assertEqual(self.get_fields('char.noreadonly'), [ID_FIELD]) + + def test_readonly_states_stillreadonly(self): + """ Readonly fields with readonly states leaving them readonly + always... filtered out""" + self.assertEqual(self.get_fields('char.stillreadonly'), [ID_FIELD]) + + def test_m2o(self): + """ M2O fields should allow import of themselves (name_get), + their id and their xid""" + self.assertEqual(self.get_fields('m2o'), make_field(fields=[ + {'id': 'value', 'name': 'id', 'string': 'External ID', 'required': False, 'fields': []}, + {'id': 'value', 'name': '.id', 'string': 'Database ID', 'required': False, 'fields': []}, + ])) + + def test_m2o_required(self): + """ If an m2o field is required, its three sub-fields are + required as well (the client has to handle that: requiredness + is id-based) + """ + self.assertEqual(self.get_fields('m2o.required'), make_field(required=True, fields=[ + {'id': 'value', 'name': 'id', 'string': 'External ID', 'required': True, 'fields': []}, + {'id': 'value', 'name': '.id', 'string': 'Database ID', 'required': True, 'fields': []}, + ])) + +class test_o2m(TransactionCase): + def get_fields(self, field): + return self.registry('base_import.import')\ + .get_fields(self.cr, self.uid, 'base_import.tests.models.' + field) + + def test_shallow(self): + self.assertEqual(self.get_fields('o2m'), make_field(fields=[ + {'id': 'id', 'name': 'id', 'string': 'External ID', 'required': False, 'fields': []}, + # FIXME: should reverse field be ignored? + {'id': 'parent_id', 'name': 'parent_id', 'string': 'unknown', 'required': False, 'fields': [ + {'id': 'parent_id', 'name': 'id', 'string': 'External ID', 'required': False, 'fields': []}, + {'id': 'parent_id', 'name': '.id', 'string': 'Database ID', 'required': False, 'fields': []}, + ]}, + {'id': 'value', 'name': 'value', 'string': 'unknown', 'required': False, 'fields': []}, + ]))