From 680214c47e80f30d2fd62cd56dfd6169dd9f2fbb Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Wed, 13 Aug 2014 22:59:13 +0200 Subject: [PATCH] [FIX] tools.translate: when loading entries from a PO file, use also the POT. On Launchpad, as commented on bug 933496, PO entries (and their comments) are shared between series. This mean that e.g. the 7.0 series can have the wrong `reference` comments (those beginning with #:) as they would be copied from say the trunk series. Those `reference` comments are used to import translations and look them up. This patch adds a few lines of code to tools.translate so that targets defined in the POT `reference` comments are used in addition to those from the PO file. Also adds a test module to validate the new behavior. This patch stems from: - the 6.1 branch by Vo Minh Thu: https://code.launchpad.net/+branch/~openerp-dev/openobject-server/6.1-fix-po-targets-933496-vmt - the 7.0 port by Numerigraphe: https://code.launchpad.net/~numerigraphe-team/openobject-server/7.0-fix-po-targets-933496-vmt --- .../test_translation_import/__init__.py | 2 + .../test_translation_import/__openerp__.py | 15 +++++ .../addons/test_translation_import/i18n/fr.po | 52 ++++++++++++++ .../i18n/test_translation_import.pot | 58 ++++++++++++++++ .../addons/test_translation_import/models.py | 20 ++++++ .../addons/test_translation_import/tests.yml | 13 ++++ .../test_translation_import/tests/__init__.py | 9 +++ .../tests/test_term_count.py | 16 +++++ .../addons/test_translation_import/view.xml | 23 +++++++ openerp/tools/translate.py | 67 +++++++++++++++++-- 10 files changed, 270 insertions(+), 5 deletions(-) create mode 100644 openerp/tests/addons/test_translation_import/__init__.py create mode 100644 openerp/tests/addons/test_translation_import/__openerp__.py create mode 100644 openerp/tests/addons/test_translation_import/i18n/fr.po create mode 100644 openerp/tests/addons/test_translation_import/i18n/test_translation_import.pot create mode 100644 openerp/tests/addons/test_translation_import/models.py create mode 100644 openerp/tests/addons/test_translation_import/tests.yml create mode 100644 openerp/tests/addons/test_translation_import/tests/__init__.py create mode 100644 openerp/tests/addons/test_translation_import/tests/test_term_count.py create mode 100644 openerp/tests/addons/test_translation_import/view.xml diff --git a/openerp/tests/addons/test_translation_import/__init__.py b/openerp/tests/addons/test_translation_import/__init__.py new file mode 100644 index 00000000000..89d26e2f597 --- /dev/null +++ b/openerp/tests/addons/test_translation_import/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +import models diff --git a/openerp/tests/addons/test_translation_import/__openerp__.py b/openerp/tests/addons/test_translation_import/__openerp__.py new file mode 100644 index 00000000000..29c3c596396 --- /dev/null +++ b/openerp/tests/addons/test_translation_import/__openerp__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +{ + 'name': 'test-translation-import', + 'version': '0.1', + 'category': 'Tests', + 'description': """A module to test translation import.""", + 'author': 'OpenERP SA', + 'maintainer': 'OpenERP SA', + 'website': 'http://www.openerp.com', + 'depends': ['base'], + 'data': ['view.xml'], + 'test': ['tests.yml'], + 'installable': True, + 'auto_install': False, +} diff --git a/openerp/tests/addons/test_translation_import/i18n/fr.po b/openerp/tests/addons/test_translation_import/i18n/fr.po new file mode 100644 index 00000000000..f179cc1eefd --- /dev/null +++ b/openerp/tests/addons/test_translation_import/i18n/fr.po @@ -0,0 +1,52 @@ +# This is a test PO file, not a true french translation. +# See the POT file for further information. +msgid "" +msgstr "" +"Project-Id-Version: OpenERP Server 6.1\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2012-10-17 12:36+0000\n" +"PO-Revision-Date: 2012-10-17 12:36+0000\n" +"Last-Translator: <>\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +# Note: there is normally an additional line: +# #: code:addons/test_translation_import/models.py:17 +# This line is present in the POT and removed here to test the translation +# import behavior. +#. module: test_translation_import +#: field:test.translation.import,name:0 +#, python-format +msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB" +msgstr "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB in french" + +#. module: test_translation_import +#: code:addons/test_translation_import/models.py:14 +#, python-format +msgid "Ijkl" +msgstr "Ijkl in french" + +#. module: test_translation_import +#: model:ir.model,name:test_translation_import.model_test_translation_import +msgid "test.translation.import" +msgstr "test.translation.import in french" + +#. module: test_translation_import +#: help:test.translation.import,name:0 +msgid "Efgh" +msgstr "Efgh in french" + +#. module: test_translation_import +#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import +#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import +msgid "Test translation import" +msgstr "Test translation import in french" + +#. module: test_translation_import +#: model:ir.ui.menu,name:test_translation_import.menu_test_translation +msgid "Test translation" +msgstr "Test translation in french" + diff --git a/openerp/tests/addons/test_translation_import/i18n/test_translation_import.pot b/openerp/tests/addons/test_translation_import/i18n/test_translation_import.pot new file mode 100644 index 00000000000..034fa5254dc --- /dev/null +++ b/openerp/tests/addons/test_translation_import/i18n/test_translation_import.pot @@ -0,0 +1,58 @@ +# This is a test POT file, not a true template. It is manually maintained +# to test the import translation behavior of OpenERP. +# +# In particular, the +# `1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB` source is +# given with two targets (the #: comments): `code` and `field`. The code one is +# removed in the fr.po file. Still, the import should generate a database entry +# for the `code` one. I.e. the targets defined in the POT must be added to the +# targets defined in the PO file. This was done to fix a bug, as reported by +# lp:933496. +# +msgid "" +msgstr "" +"Project-Id-Version: OpenERP Server 6.1\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2012-10-17 12:36+0000\n" +"PO-Revision-Date: 2012-10-17 12:36+0000\n" +"Last-Translator: <>\n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: test_translation_import +#: code:addons/test_translation_import/models.py:17 +#: field:test.translation.import,name:0 +#, python-format +msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB" +msgstr "" + +#. module: test_translation_import +#: code:addons/test_translation_import/models.py:14 +#, python-format +msgid "Ijkl" +msgstr "" + +#. module: test_translation_import +#: model:ir.model,name:test_translation_import.model_test_translation_import +msgid "test.translation.import" +msgstr "" + +#. module: test_translation_import +#: help:test.translation.import,name:0 +msgid "Efgh" +msgstr "" + +#. module: test_translation_import +#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import +#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import +msgid "Test translation import" +msgstr "" + +#. module: test_translation_import +#: model:ir.ui.menu,name:test_translation_import.menu_test_translation +msgid "Test translation" +msgstr "" + diff --git a/openerp/tests/addons/test_translation_import/models.py b/openerp/tests/addons/test_translation_import/models.py new file mode 100644 index 00000000000..3dc3f1c71cc --- /dev/null +++ b/openerp/tests/addons/test_translation_import/models.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +import openerp +from openerp.tools.translate import _ + +class m(openerp.osv.orm.TransientModel): + """ A model to provide source strings. + """ + _name = 'test.translation.import' + + _columns = { + 'name': openerp.osv.fields.char( + '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB', + size=32, help='Efgh'), + } + + _('Ijkl') + + # With the name label above, this source string should be generated twice. + _('1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB') + diff --git a/openerp/tests/addons/test_translation_import/tests.yml b/openerp/tests/addons/test_translation_import/tests.yml new file mode 100644 index 00000000000..8ccae43f6ee --- /dev/null +++ b/openerp/tests/addons/test_translation_import/tests.yml @@ -0,0 +1,13 @@ +- + Load the french translation. +- + !python {model: ir.translation }: | + import openerp + openerp.tools.trans_load(cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False) +- + Assert we have loaded the correct number of entries for the given source string. +- + !python {model: ir.translation }: | + ids = self.search(cr, uid, + [('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB')]) + assert len(ids) == 2, "2 entries are expected, got %s instead." % len(ids) diff --git a/openerp/tests/addons/test_translation_import/tests/__init__.py b/openerp/tests/addons/test_translation_import/tests/__init__.py new file mode 100644 index 00000000000..09e712b0f80 --- /dev/null +++ b/openerp/tests/addons/test_translation_import/tests/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +import unittest2 + +import test_term_count + +suite = [ + test_term_count + ] + diff --git a/openerp/tests/addons/test_translation_import/tests/test_term_count.py b/openerp/tests/addons/test_translation_import/tests/test_term_count.py new file mode 100644 index 00000000000..2d065b05f18 --- /dev/null +++ b/openerp/tests/addons/test_translation_import/tests/test_term_count.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +import openerp +from openerp.tests import common + +class TestTermCount(common.TransactionCase): + + def test_count_term(self): + """ + Just make sure we have as many translation entries as we wanted. + """ + openerp.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False) + ids = self.registry('ir.translation').search(self.cr, self.uid, + [('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB')]) + self.assertEqual(len(ids), 2) + diff --git a/openerp/tests/addons/test_translation_import/view.xml b/openerp/tests/addons/test_translation_import/view.xml new file mode 100644 index 00000000000..2fb26fbbdc5 --- /dev/null +++ b/openerp/tests/addons/test_translation_import/view.xml @@ -0,0 +1,23 @@ + + + + + + Test translation import + ir.actions.act_window + test.translation.import + form + tree,form + current + + + + + + + + + diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index b3bea678cf8..529f3b916c4 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -313,6 +313,10 @@ class TinyPoFile(object): if not line.startswith('module:'): comments.append(line) elif line.startswith('#:'): + # Process the `reference` comments. Each line can specify + # multiple targets (e.g. model, view, code, selection, + # ...). For each target, we will return an additional + # entry. for lpart in line[2:].strip().split(' '): trans_info = lpart.strip().split(':',2) if trans_info and len(trans_info) == 2: @@ -362,6 +366,9 @@ class TinyPoFile(object): line = self.lines.pop(0).strip() if targets and not fuzzy: + # Use the first target for the current entry (returned at the + # end of this next() call), and keep the others to generate + # additional entries (returned the next next() calls). trans_type, name, res_id = targets.pop(0) for t, n, r in targets: if t == trans_type == 'code': continue @@ -945,6 +952,10 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, # lets create the language with locale information lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name) + # Parse also the POT: it will possibly provide additional targets. + # (Because the POT comments are correct on Launchpad but not the + # PO comments due to a Launchpad limitation. See LP bug 933496.) + pot_reader = [] # now, the serious things: we read the language file fileobj.seek(0) @@ -957,19 +968,42 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, elif fileformat == 'po': reader = TinyPoFile(fileobj) f = ['type', 'name', 'res_id', 'src', 'value', 'comments'] + + # Make a reader for the POT file and be somewhat defensive for the + # stable branch. + if fileobj.name.endswith('.po'): + try: + # Normally the path looks like /path/to/xxx/i18n/lang.po + # and we try to find the corresponding + # /path/to/xxx/i18n/xxx.pot file. + head, _ = os.path.split(fileobj.name) + head2, _ = os.path.split(head) + head3, tail3 = os.path.split(head2) + pot_handle = misc.file_open(os.path.join(head3, tail3, 'i18n', tail3 + '.pot')) + pot_reader = TinyPoFile(pot_handle) + except: + pass + else: _logger.error('Bad file format: %s', fileformat) raise Exception(_('Bad file format')) + # Read the POT `reference` comments, and keep them indexed by source + # string. + pot_targets = {} + for type, name, res_id, src, _, comments in pot_reader: + if type is not None: + pot_targets.setdefault(src, {'value': None, 'targets': []}) + pot_targets[src]['targets'].append((type, name, res_id)) + # read the rest of the file - line = 1 irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context) - for row in reader: - line += 1 + def process_row(row): + """Process a single PO (or POT) entry.""" # skip empty rows and rows where the translation field (=last fiefd) is empty #if (not row) or (not row[-1]): - # continue + # return # dictionary which holds values for this line of the csv file # {'lang': ..., 'type': ..., 'name': ..., 'res_id': ..., @@ -979,9 +1013,17 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, for i, field in enumerate(f): dic[field] = row[i] + # Get the `reference` comments from the POT. + src = row[3] + if pot_reader and src in pot_targets: + pot_targets[src]['targets'] = filter(lambda x: x != row[:3], pot_targets[src]['targets']) + pot_targets[src]['value'] = row[4] + if not pot_targets[src]['targets']: + del pot_targets[src] + # This would skip terms that fail to specify a res_id if not dic.get('res_id'): - continue + return res_id = dic.pop('res_id') if res_id and isinstance(res_id, (int, long)) \ @@ -1002,6 +1044,21 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, irt_cursor.push(dic) + # First process the entries from the PO file (doing so also fills/removes + # the entries from the POT file). + for row in reader: + process_row(row) + + # Then process the entries implied by the POT file (which is more + # correct w.r.t. the targets) if some of them remain. + pot_rows = [] + for src in pot_targets: + value = pot_targets[src]['value'] + for type, name, res_id in pot_targets[src]['targets']: + pot_rows.append((type, name, res_id, src, value, comments)) + for row in pot_rows: + process_row(row) + irt_cursor.finish() trans_obj.clear_caches() if verbose: