From c2402b14a9011fc4d24557b270421d2f3ccb574c Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Wed, 5 Jan 2011 15:07:58 +0100 Subject: [PATCH 1/5] [IMP] trans_export/trans_load: passing a cursor around instead of a dbname. bzr revid: vmt@openerp.com-20110105140758-vgoufkd85v6ywrjk --- bin/addons/base/module/module.py | 4 ++-- .../module/wizard/base_import_language.py | 4 ++-- .../module/wizard/base_update_translations.py | 6 ++--- bin/openerp-server.py | 11 +++++++-- bin/tools/translate.py | 23 +++++++------------ 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/bin/addons/base/module/module.py b/bin/addons/base/module/module.py index d90ef415951..3237da178cc 100644 --- a/bin/addons/base/module/module.py +++ b/bin/addons/base/module/module.py @@ -501,7 +501,7 @@ class module(osv.osv): f2 = addons.get_module_resource(mod.name, 'i18n', iso_lang2 + '.po') if f2: logger.info('module %s: loading base translation file %s for language %s', mod.name, iso_lang2, lang) - tools.trans_load(cr.dbname, f2, lang, verbose=False, context=context) + tools.trans_load(cr, f2, lang, verbose=False, context=context) context2['overwrite'] = True # Implementation notice: we must first search for the full name of # the language derivative, like "en_UK", and then the generic, @@ -511,7 +511,7 @@ class module(osv.osv): f = addons.get_module_resource(mod.name, 'i18n', iso_lang + '.po') if f: logger.info('module %s: loading translation file (%s) for language %s', mod.name, iso_lang, lang) - tools.trans_load(cr.dbname, f, lang, verbose=False, context=context2) + tools.trans_load(cr, f, lang, verbose=False, context=context2) elif iso_lang != 'en': logger.warning('module %s: no translation for language %s', mod.name, iso_lang) diff --git a/bin/addons/base/module/wizard/base_import_language.py b/bin/addons/base/module/wizard/base_import_language.py index 6608ade03ef..96dcd1df794 100644 --- a/bin/addons/base/module/wizard/base_import_language.py +++ b/bin/addons/base/module/wizard/base_import_language.py @@ -56,8 +56,8 @@ class base_language_import(osv.osv_memory): fileformat = first_line.endswith("type,name,res_id,src,value") and 'csv' or 'po' fileobj.seek(0) - tools.trans_load_data(cr.dbname, fileobj, fileformat, import_data.code, lang_name=import_data.name) + tools.trans_load_data(cr, fileobj, fileformat, import_data.code, lang_name=import_data.name) fileobj.close() return {} -base_language_import() \ No newline at end of file +base_language_import() diff --git a/bin/addons/base/module/wizard/base_update_translations.py b/bin/addons/base/module/wizard/base_update_translations.py index 6b329cf1212..f79f4c0f1fa 100644 --- a/bin/addons/base/module/wizard/base_update_translations.py +++ b/bin/addons/base/module/wizard/base_update_translations.py @@ -45,8 +45,8 @@ class base_update_translations(osv.osv_memory): this = self.browse(cr, uid, ids)[0] lang_name = self._get_lang_name(cr, uid, this.lang) buf=cStringIO.StringIO() - tools.trans_export(this.lang, ['all'], buf, 'csv', dbname=cr.dbname) - tools.trans_load_data(cr.dbname, buf, 'csv', this.lang, lang_name=lang_name) + tools.trans_export(this.lang, ['all'], buf, 'csv', cr) + tools.trans_load_data(cr, buf, 'csv', this.lang, lang_name=lang_name) buf.close() return {'type': 'ir.actions.act_window_close'} @@ -70,4 +70,4 @@ class base_update_translations(osv.osv_memory): 'lang': fields.selection(_get_languages, 'Language', required=True), } -base_update_translations() \ No newline at end of file +base_update_translations() diff --git a/bin/openerp-server.py b/bin/openerp-server.py index 5c67c879d84..3ffa2a3e7d3 100755 --- a/bin/openerp-server.py +++ b/bin/openerp-server.py @@ -144,7 +144,10 @@ if tools.config["translate_out"]: fileformat = os.path.splitext(tools.config["translate_out"])[-1][1:].lower() buf = file(tools.config["translate_out"], "w") - tools.trans_export(tools.config["language"], tools.config["translate_modules"], buf, fileformat) + dbname = tools.config['db_name'] + cr = pooler.get_db(dbname).cursor() + tools.trans_export(tools.config["language"], tools.config["translate_modules"] or ["all"], buf, fileformat, cr) + cr.close() buf.close() logger.info('translation file written successfully') @@ -152,10 +155,14 @@ if tools.config["translate_out"]: if tools.config["translate_in"]: context = {'overwrite': tools.config["overwrite_existing_translations"]} - tools.trans_load(tools.config["db_name"], + dbname = tools.config['db_name'] + cr = pooler.get_db(dbname).cursor() + tools.trans_load(cr, tools.config["translate_in"], tools.config["language"], context=context) + cr.commit() + cr.close() sys.exit(0) #---------------------------------------------------------------------------------- diff --git a/bin/tools/translate.py b/bin/tools/translate.py index c1c09140d0a..bc1ee121b91 100644 --- a/bin/tools/translate.py +++ b/bin/tools/translate.py @@ -409,7 +409,7 @@ class TinyPoFile(object): # Methods to export the translation file -def trans_export(lang, modules, buffer, format, dbname=None): +def trans_export(lang, modules, buffer, format, cr): def _process(format, modules, rows, buffer, lang, newlang): if format == 'csv': @@ -461,7 +461,7 @@ def trans_export(lang, modules, buffer, format, dbname=None): newlang = not bool(lang) if newlang: lang = 'en_US' - trans = trans_generate(lang, modules, dbname) + trans = trans_generate(lang, modules, cr) if newlang and format!='csv': for trx in trans: trx[-1] = '' @@ -524,17 +524,13 @@ def in_modules(object_name, modules): module = module_dict.get(module, module) return module in modules -def trans_generate(lang, modules, dbname=None): +def trans_generate(lang, modules, cr): logger = logging.getLogger('i18n') - if not dbname: - dbname=tools.config['db_name'] - if not modules: - modules = ['all'] + dbname = cr.dbname pool = pooler.get_pool(dbname) trans_obj = pool.get('ir.translation') model_data_obj = pool.get('ir.model.data') - cr = pooler.get_db(dbname).cursor() uid = 1 l = pool.obj_pool.items() l.sort() @@ -834,16 +830,15 @@ def trans_generate(lang, modules, dbname=None): trans = trans_obj._get_source(cr, uid, name, type, lang, source) out.append([module, type, name, id, source, encode(trans) or '']) - cr.close() return out -def trans_load(db_name, filename, lang, verbose=True, context=None): +def trans_load(cr, filename, lang, verbose=True, context=None): logger = logging.getLogger('i18n') try: fileobj = open(filename,'r') logger.info("loading %s", filename) fileformat = os.path.splitext(filename)[-1][1:].lower() - r = trans_load_data(db_name, fileobj, fileformat, lang, verbose=verbose, context=context) + r = trans_load_data(cr, fileobj, fileformat, lang, verbose=verbose, context=context) fileobj.close() return r except IOError: @@ -851,12 +846,13 @@ def trans_load(db_name, filename, lang, verbose=True, context=None): logger.error("couldn't read translation file %s", filename) return None -def trans_load_data(db_name, fileobj, fileformat, lang, lang_name=None, verbose=True, context=None): +def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, context=None): logger = logging.getLogger('i18n') if verbose: logger.info('loading translation file for language %s', lang) if context is None: context = {} + db_name = cr.dbname pool = pooler.get_pool(db_name) lang_obj = pool.get('res.lang') trans_obj = pool.get('ir.translation') @@ -864,7 +860,6 @@ def trans_load_data(db_name, fileobj, fileformat, lang, lang_name=None, verbose= iso_lang = tools.get_iso_codes(lang) try: uid = 1 - cr = pooler.get_db(db_name).cursor() ids = lang_obj.search(cr, uid, [('code','=', lang)]) if not ids: @@ -967,8 +962,6 @@ def trans_load_data(db_name, fileobj, fileformat, lang, lang_name=None, verbose= trans_obj.write(cr, uid, ids, {'value': dic['value']}) else: trans_obj.create(cr, uid, dic) - cr.commit() - cr.close() if verbose: logger.info("translation file loaded succesfully") except IOError: From fb52e889577e333daef37e610fbabc2230b242af Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Wed, 5 Jan 2011 16:16:58 +0100 Subject: [PATCH 2/5] [IMP] test_translate: code and yml to test the translation mechanism. bzr revid: vmt@openerp.com-20110105151658-973xp0pyizisuyv5 --- bin/addons/base/__openerp__.py | 1 + bin/addons/base/test/test_translation.yml | 15 ++++ bin/tools/test_translate.py | 94 +++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 bin/addons/base/test/test_translation.yml create mode 100644 bin/tools/test_translate.py diff --git a/bin/addons/base/__openerp__.py b/bin/addons/base/__openerp__.py index bb419fc90d3..1c94d6b90b8 100644 --- a/bin/addons/base/__openerp__.py +++ b/bin/addons/base/__openerp__.py @@ -90,6 +90,7 @@ 'test/test_context.xml', 'test/bug_lp541545.xml', 'test/test_osv_expression.yml', + 'test/test_translation.yml', ], 'installable': True, 'active': True, diff --git a/bin/addons/base/test/test_translation.yml b/bin/addons/base/test/test_translation.yml new file mode 100644 index 00000000000..602e3ea3d7c --- /dev/null +++ b/bin/addons/base/test/test_translation.yml @@ -0,0 +1,15 @@ +- | + Test the translation mechanism by synchronizing a dummy language and + checking its views. +- + Install dummy language xx_XX +- + !python {model: base.language.install }: | + import tools.test_translate + tools.test_translate.install_dummy_language(cr, uid) +- + Check views using dummy language xx_XX +- + !python {model: res.lang }: | + import tools.test_translate + tools.test_translate.check_all_views(cr, uid) diff --git a/bin/tools/test_translate.py b/bin/tools/test_translate.py new file mode 100644 index 00000000000..245473859c4 --- /dev/null +++ b/bin/tools/test_translate.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2011 OpenERP s.a. (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" Helper functions for translation testing. +""" + +import pooler +import logging + +def install_dummy_language(cr, uid): + """ Install a dummy language xx_XX and synchronize it (create all terms). + """ + + pool = pooler.get_pool(cr.dbname) + res_lang = pool.get('res.lang') + base_update_translations = pool.get('base.update.translations') + + # create xx_XX if it doesn't exist yet. + ids = res_lang.search(cr, uid, [('code', '=', 'xx_XX')]) + if not ids: + res_lang.create(cr, uid, {'code': 'xx_XX', 'iso_code': 'xx', 'name': 'Dummy Language', 'translatable': True}) + + # create base.update.translations if it doesn't exist yet. + ids = base_update_translations.search(cr, uid, [('lang', '=', 'xx_XX')]) + id = 0 + if not ids: + id = base_update_translations.create(cr, uid, {'lang': 'xx_XX'}) + else: + id = ids[0] + + # synchronize xx_XX + print base_update_translations.act_update(cr, uid, [id]) + + # add marks on all xx_XX terms + cr.execute(u"update ir_translation set value='⟨⟨'||value||'⟩⟩' where lang='xx_XX'") + +def has_marks(s): + return s[:2] == u"⟨⟨" and s[-2:] == u"⟩⟩" + +def check_all_views(cr, uid): + pool = pooler.get_pool(cr.dbname) + ir_ui_view = pool.get('ir.ui.view') + + ids = ir_ui_view.search(cr, uid, []) + for view in ir_ui_view.browse(cr, uid, ids): + check_view(cr, uid, view.id, view.model) + +def check_view(cr, uid, view_id, view_model): + pool = pooler.get_pool(cr.dbname) + model = pool.get(view_model) + if not model: + return + + log = logging.getLogger('tools.test_translate') + + o = model.fields_view_get(cr, uid, view_id, 'tree', {'lang': 'xx_XX'}) + + arch = o['arch'] + # TODO check translation in arch (see trans_parse_view in bin/tools/translate.py, + # search views, ... + # TODO it happens that o['view_id'] != view_id... + fields = o['fields'] + checked_fields = 0 + for k, v in fields.items(): + checked_fields = checked_fields + 1 + if v.has_key('string') and not has_marks(v['string']): + print "string not translated in %s / %s / %s (view_id: %s)" % (o['model'], o['name'], o['type'], o['view_id']) + if v.has_key('help') and not has_marks(v['help']): + print "help not translated in %s / %s / %s (view_id: %s)" % (o['model'], o['name'], o['type'], o['view_id']) + if v.has_key('selection'): + for selk, selv in v['selection']: + if not has_marks(selv): + print "selection not translated in %s / %s / %s (view_id: %s)" % (o['model'], o['name'], o['type'], o['view_id']) + break + #print "%s fields checked for %s %s %s %s %s" % (checked_fields, o['model'], o['name'], o['type'], o['view_id'], view_id) + From 4629c7af7838304d686b6c59197b2b1ac981704f Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 6 Jan 2011 11:50:21 +0100 Subject: [PATCH 3/5] [IMP] translation: Added two columns to ir_translation to fix the res_ids after loading is done. Currently res_ids are resolved and written to ir_translation on the fly, as the .po are loaded, line by line. Doing this in a second step should resolve a translation bug and make things faster too. bzr revid: vmt@openerp.com-20110106105021-xj9vah5oz0ceto7i --- bin/addons/base/ir/ir_translation.py | 7 ++++++- bin/tools/translate.py | 30 ++++++++++++++++++---------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/bin/addons/base/ir/ir_translation.py b/bin/addons/base/ir/ir_translation.py index eec5b8522bb..a19a355540a 100644 --- a/bin/addons/base/ir/ir_translation.py +++ b/bin/addons/base/ir/ir_translation.py @@ -61,6 +61,8 @@ class ir_translation(osv.osv): 'type': fields.selection(TRANSLATION_TYPE, string='Type', size=16, select=True), 'src': fields.text('Source'), 'value': fields.text('Translation Value'), + 'module': fields.char('Module', size=64), # TODO foreign key to ir_model_data + 'xml_id': fields.char('XML Id', size=128), # idem } def _auto_init(self, cr, context={}): @@ -88,7 +90,10 @@ class ir_translation(osv.osv): cr.execute('CREATE INDEX ir_translation_ltn ON ir_translation (name, lang, type)') cr.commit() - + cr.execute('SELECT indexname FROM pg_indexes WHERE indexname = %s', ('ir_translation_mx',)) + if not cr.fetchone(): + cr.execute('CREATE INDEX ir_translation_mx ON ir_translation (module, xml_id)') + cr.commit() @tools.cache(skiparg=3, multi='ids') def _get_ids(self, cr, uid, name, tt, lang, ids): diff --git a/bin/tools/translate.py b/bin/tools/translate.py index bc1ee121b91..363dc3b05ea 100644 --- a/bin/tools/translate.py +++ b/bin/tools/translate.py @@ -846,6 +846,9 @@ def trans_load(cr, filename, lang, verbose=True, context=None): logger.error("couldn't read translation file %s", filename) return None +# Populates the ir_translation table. Fixing the res_ids so that they point +# correctly to ir_model_data is done in a separate step, using the +# 'trans_update_res_ids' function below. def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, context=None): logger = logging.getLogger('i18n') if verbose: @@ -936,17 +939,13 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, try: dic['res_id'] = dic['res_id'] and int(dic['res_id']) or 0 + dic['module'] = False + dic['xml_id'] = False except: - model_data_ids = model_data_obj.search(cr, uid, [ - ('model', '=', dic['name'].split(',')[0]), - ('module', '=', dic['res_id'].split('.', 1)[0]), - ('name', '=', dic['res_id'].split('.', 1)[1]), - ]) - if model_data_ids: - dic['res_id'] = model_data_obj.browse(cr, uid, - model_data_ids[0]).res_id - else: - dic['res_id'] = False + splitted = dic['res_id'].split('.', 1) + dic['module'] = splitted[0] + dic['xml_id'] = splitted[1] + dic['res_id'] = False args = [ ('lang', '=', lang), @@ -968,6 +967,17 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True, filename = '[lang: %s][format: %s]' % (iso_lang or 'new', fileformat) logger.exception("couldn't read translation file %s", filename) +def trans_update_res_ids(cr): + cr.execute("""\ +update ir_translation +set res_id = ir_model_data.res_id +from ir_model_data +where ir_translation.module = ir_model_data.module +and ir_translation.xml_id = ir_model_data.name +and ir_translation.module is not null +and ir_translation.xml_id is not null; + """) + def get_locales(lang=None): if lang is None: lang = locale.getdefaultlocale()[0] From fbc332935b6cd6cae38acd44ceabf21a9cae2bee Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 6 Jan 2011 14:06:54 +0100 Subject: [PATCH 4/5] [IMP] translation: added trans_update_res_ids to resolve the res_ids from ir_translation. bzr revid: vmt@openerp.com-20110106130654-tp8jyjo3h3nvgqbq --- bin/addons/base/module/module.py | 1 + bin/addons/base/module/wizard/base_import_language.py | 1 + bin/addons/base/module/wizard/base_update_translations.py | 1 + bin/openerp-server.py | 1 + 4 files changed, 4 insertions(+) diff --git a/bin/addons/base/module/module.py b/bin/addons/base/module/module.py index 3237da178cc..d050355ab05 100644 --- a/bin/addons/base/module/module.py +++ b/bin/addons/base/module/module.py @@ -514,6 +514,7 @@ class module(osv.osv): tools.trans_load(cr, f, lang, verbose=False, context=context2) elif iso_lang != 'en': logger.warning('module %s: no translation for language %s', mod.name, iso_lang) + tools.trans_update_res_ids(cr) def check(self, cr, uid, ids, context=None): logger = logging.getLogger('init') diff --git a/bin/addons/base/module/wizard/base_import_language.py b/bin/addons/base/module/wizard/base_import_language.py index 96dcd1df794..d73cd0864a6 100644 --- a/bin/addons/base/module/wizard/base_import_language.py +++ b/bin/addons/base/module/wizard/base_import_language.py @@ -57,6 +57,7 @@ class base_language_import(osv.osv_memory): fileobj.seek(0) tools.trans_load_data(cr, fileobj, fileformat, import_data.code, lang_name=import_data.name) + tools.trans_update_res_ids(cr) fileobj.close() return {} diff --git a/bin/addons/base/module/wizard/base_update_translations.py b/bin/addons/base/module/wizard/base_update_translations.py index f79f4c0f1fa..bd950ea628d 100644 --- a/bin/addons/base/module/wizard/base_update_translations.py +++ b/bin/addons/base/module/wizard/base_update_translations.py @@ -47,6 +47,7 @@ class base_update_translations(osv.osv_memory): buf=cStringIO.StringIO() tools.trans_export(this.lang, ['all'], buf, 'csv', cr) tools.trans_load_data(cr, buf, 'csv', this.lang, lang_name=lang_name) + tools.trans_update_res_ids(cr) buf.close() return {'type': 'ir.actions.act_window_close'} diff --git a/bin/openerp-server.py b/bin/openerp-server.py index 3ffa2a3e7d3..a3f8ed84b34 100755 --- a/bin/openerp-server.py +++ b/bin/openerp-server.py @@ -161,6 +161,7 @@ if tools.config["translate_in"]: tools.config["translate_in"], tools.config["language"], context=context) + tools.trans_update_res_ids(cr) cr.commit() cr.close() sys.exit(0) From 052f6caeaa56f3d44eb469d8aa8f9ac966a3e597 Mon Sep 17 00:00:00 2001 From: Vo Minh Thu Date: Thu, 6 Jan 2011 14:28:30 +0100 Subject: [PATCH 5/5] [IMP] ir_translation: added help messages and comments. bzr revid: vmt@openerp.com-20110106132830-l0jynvpdd49xurm6 --- bin/addons/base/ir/ir_translation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/addons/base/ir/ir_translation.py b/bin/addons/base/ir/ir_translation.py index a19a355540a..de4fd527a60 100644 --- a/bin/addons/base/ir/ir_translation.py +++ b/bin/addons/base/ir/ir_translation.py @@ -61,8 +61,10 @@ class ir_translation(osv.osv): 'type': fields.selection(TRANSLATION_TYPE, string='Type', size=16, select=True), 'src': fields.text('Source'), 'value': fields.text('Translation Value'), - 'module': fields.char('Module', size=64), # TODO foreign key to ir_model_data - 'xml_id': fields.char('XML Id', size=128), # idem + # These two columns map to ir_model_data.module and ir_model_data.name. + # They are used to resolve the res_id above after loading is done. + 'module': fields.char('Module', size=64, help='Maps to the ir_model_data for which this translation is provided.'), + 'xml_id': fields.char('XML Id', size=128, help='Maps to the ir_model_data for which this translation is provided.'), } def _auto_init(self, cr, context={}):