[MERGE] [FIX] res_font: make it works in multiworker environment

res_font model gains family, path and mode field
Basic fonts embedded in pdf (Helvetica, Times and Courier) are data
Fix reports using font names to be valid (eg: DejaVu Sans Bold -> DejaVuSans-Bold)
default_get of res_company does not scan the filesystem anymore
family field is new name_get font for res_font (more user friendly)
Domain on displayed font to only 'normal ones' (not bold or italic)
Handle fonts in 'book' mode as normal in setTTFontMapping
Relax constraint on res_font to unique family and name ('name' was actually family before)
font_scan is called on 'reload fonts' action button and for each report rendering
font_scan is lazy when loaded from report and scanning the filesystem only if no font recorded in filesystem
'/dev/null' in font path is used to add fonts in the list but not loaded by reportlab
Scanning the filesystem triggers clear_caches
clear_cache resets the list of found fonts to empty list to make font_scan call to reload the full list (when redering a report on any worker, will always get the updated font list after a scan)
Add access rights for fonts (read everybody, create admin, call font_scan as superuser in report rendering)
Remove font hack with BaseCustomTTFonts, rely fully on the system fonts
Use glob instead of listdir (needed for path like '/usr/share/fonts/TTF/*')
Make warning message from reportlab more self-explaining.

Also make the runbot green (which is nice)

bzr revid: mat@openerp.com-20131209171338-o796pldia9da3kt2

openerp/addons/base/base_data.xml

@@ -91,5 +91,25 @@ Administrator</field>
             <field eval="10" name="sequence"/>
         </record>
 
+        <!-- Basic fonts family included in PDF standart, will always be in the font list -->
+        <record model="res.font" id="base.font_helvetica">
+            <field name="name">Helvetica</field>
+            <field name="family">Helvetica</field>
+            <field name="path">/dev/null</field>
+            <field name="mode">all</field>
+        </record>
+        <record model="res.font" id="base.font_times">
+            <field name="name">Times</field>
+            <field name="family">Times</field>
+            <field name="path">/dev/null</field>
+            <field name="mode">all</field>
+        </record>
+        <record model="res.font" id="base.font_courier">
+            <field name="name">Courier</field>
+            <field name="family">Courier</field>
+            <field name="path">/dev/null</field>
+            <field name="mode">all</field>
+        </record>
+
     </data>
 </openerp>

openerp/addons/base/report/custom_default_printscreen.xsl

@@ -5,9 +5,9 @@
 		<!--logo-->
 		<fill color="black"/>
         <stroke color="black"/>
-        <setFont name="DejaVu Sans" size="8"/>
+        <setFont name="DejaVuSans" size="8"/>
         <drawString x="1.3cm" y="19.5cm"><xsl:value-of select="//header-date"/></drawString>
-        <setFont name="DejaVu Sans Bold" size="10"/>
+        <setFont name="DejaVuSans-Bold" size="10"/>
         <drawString x="13.8cm" y="19.5cm"><xsl:value-of select="//company"/></drawString>
         <stroke color="#000000"/>
         <lines size="8">1.3cm 19.3cm 28.5cm 19.3cm</lines>

openerp/addons/base/res/res_company.py

@@ -27,7 +27,6 @@ from openerp.osv import fields, osv
 from openerp.tools.translate import _
 from openerp.tools.safe_eval import safe_eval as eval
 from openerp.tools import image_resize_image
-from openerp.report.render.rml2pdf import customfonts
   
 class multi_company_default(osv.osv):
     """
@@ -183,17 +182,20 @@ class res_company(osv.osv):
         
     def onchange_font_name(self, cr, uid, ids, font, rml_header, rml_header2, rml_header3, context=None):
         """ To change default header style of all <para> and drawstring. """
-        
+
         def _change_header(header,font):
             """ Replace default fontname use in header and setfont tag """
             
-            default_para = re.sub('fontName.?=.?".*"', 'fontName="%s"'% font,header)
+            default_para = re.sub('fontName.?=.?".*"', 'fontName="%s"'% font, header)
-            return re.sub('(<setFont.?name.?=.?)(".*?")(.)', '\g<1>"%s"\g<3>'% font,default_para)
+            return re.sub('(<setFont.?name.?=.?)(".*?")(.)', '\g<1>"%s"\g<3>'% font, default_para)
+        
+        if not font:
+            return True
         fontname = self.pool.get('res.font').browse(cr, uid, font, context=context).name
         return {'value':{
-                        'rml_header': _change_header(rml_header,fontname),
+                        'rml_header': _change_header(rml_header, fontname),
-                        'rml_header2':_change_header(rml_header2,fontname),
+                        'rml_header2':_change_header(rml_header2, fontname),
-                        'rml_header3':_change_header(rml_header3,fontname)
+                        'rml_header3':_change_header(rml_header3, fontname)
                         }}
 
     def on_change_country(self, cr, uid, ids, country_id, context=None):
@@ -294,12 +296,8 @@ class res_company(osv.osv):
 
     def _get_font(self, cr, uid, ids):
         font_obj = self.pool.get('res.font')
-        res = font_obj.search(cr, uid, [('name', '=', 'Helvetica')], limit=1)
+        res = font_obj.search(cr, uid, [('family', '=', 'Helvetica'), ('mode', '=', 'all')], limit=1)
-        if res:
+        return res and res[0] or False       
-            return res[0]
-        
-        font_obj.init_no_scan(cr, uid)
-        return font_obj.search(cr, uid, [('name', '=', 'Helvetica')], limit=1)[0]
 
     _header = """
 <header>
@@ -307,20 +305,20 @@ class res_company(osv.osv):
     <frame id="first" x1="28.0" y1="28.0" width="%s" height="%s"/>
     <stylesheet>
        <!-- Set here the default font to use for all <para> tags -->
-       <paraStyle name='Normal' fontName="DejaVu Sans"/>
+       <paraStyle name='Normal' fontName="DejaVuSans"/>
     </stylesheet>
     <pageGraphics>
         <fill color="black"/>
         <stroke color="black"/>
-        <setFont name="DejaVu Sans" size="8"/>
+        <setFont name="DejaVuSans" size="8"/>
         <drawString x="%s" y="%s"> [[ formatLang(time.strftime("%%Y-%%m-%%d"), date=True) ]]  [[ time.strftime("%%H:%%M") ]]</drawString>
-        <setFont name="DejaVu Sans Bold" size="10"/>
+        <setFont name="DejaVuSans-Bold" size="10"/>
         <drawCentredString x="%s" y="%s">[[ company.partner_id.name ]]</drawCentredString>
         <stroke color="#000000"/>
         <lines>%s</lines>
         <!-- Set here the default font to use for all <drawString> tags -->
         <!-- don't forget to change the 2 other occurence of <setFont> above if needed --> 
-        <setFont name="DejaVu Sans" size="8"/>
+        <setFont name="DejaVuSans" size="8"/>
     </pageGraphics>
 </pageTemplate>
 </header>"""
@@ -345,13 +343,13 @@ class res_company(osv.osv):
         <frame id="first" x1="1.3cm" y1="3.0cm" height="%s" width="19.0cm"/>
          <stylesheet>
             <!-- Set here the default font to use for all <para> tags -->
-            <paraStyle name='Normal' fontName="DejaVu Sans"/>
+            <paraStyle name='Normal' fontName="DejaVuSans"/>
             <paraStyle name="main_footer" fontSize="8.0" alignment="CENTER"/>
             <paraStyle name="main_header" fontSize="8.0" leading="10" alignment="LEFT" spaceBefore="0.0" spaceAfter="0.0"/>
          </stylesheet>
         <pageGraphics>
             <!-- Set here the default font to use for all <drawString> tags -->
-            <setFont name="DejaVu Sans" size="8"/>
+            <setFont name="DejaVuSans" size="8"/>
             <!-- You Logo - Change X,Y,Width and Height -->
             <image x="1.3cm" y="%s" height="40.0" >[[ company.logo or removeParentNode('image') ]]</image>
             <fill color="black"/>
@@ -396,7 +394,7 @@ class res_company(osv.osv):
         return {'value': {'rml_header': self._header_a4}}
 
     def act_discover_fonts(self, cr, uid, ids, context=None):
-        return self.pool.get("res.font").discover_fonts(cr, uid, ids, context)
+        return self.pool.get("res.font").font_scan(cr, uid, context=context)
 
     _defaults = {
         'currency_id': _get_euro,

openerp/addons/base/res/res_company_view.xml

@@ -85,7 +85,7 @@
                                 <label for="font" />
                                 <div>
                                     <div>
-                                        <field name="font" class="oe_inline" colspan="2" on_change="onchange_font_name(font, rml_header, rml_header2, rml_header3)"/>
+                                        <field name="font" class="oe_inline" colspan="2" on_change="onchange_font_name(font, rml_header, rml_header2, rml_header3)" domain="[('mode', 'in', ('normal', 'regular', 'all', 'book'))]" />
                                         <button string="(reload fonts)" name="act_discover_fonts" type="object" class="oe_link" colspan="1"/>
                                     </div>
                                 </div>

openerp/addons/base/res/res_font.py

@@ -2,7 +2,7 @@
 ##############################################################################
 #
 #    OpenERP, Open Source Management Solution
-#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
+#    Copyright (C) 2013 OpenERP SA (<http://openerp.com>).
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU Affero General Public License as
@@ -20,6 +20,7 @@
 ##############################################################################
 
 from reportlab.pdfbase import ttfonts
+from openerp.modules.registry import RegistryManager
 from openerp.osv import fields, osv
 from openerp.report.render.rml2pdf import customfonts
 
@@ -40,57 +41,75 @@ _logger = logging.getLogger(__name__)
 class res_font(osv.Model):
     _name = "res.font"
     _description = 'Fonts available'
-    _order = 'name'
+    _order = 'family,name,id'
+    _rec_name = 'family'
 
     _columns = {
-        'name': fields.char("Name", required=True),
+        'family': fields.char("Font family", required=True),
+        'name': fields.char("Font Name", required=True),
+        'path': fields.char("Path", required=True),
+        'mode': fields.char("Mode", required=True),
     }
 
     _sql_constraints = [
-        ('name_font_uniq', 'unique(name)', 'You can not register two fonts with the same name'),
+        ('name_font_uniq', 'unique(family, name)', 'You can not register two fonts with the same name'),
     ]
 
-    def discover_fonts(self, cr, uid, ids, context=None):
+    def font_scan(self, cr, uid, lazy=False, context=None):
-        """Scan fonts on the file system, add them to the list of known fonts
+        """Action of loading fonts
-        and create font object for the new ones"""
+        In lazy mode will scan the filesystem only if there is no founts in the database and sync if no font in CustomTTFonts
-        customfonts.CustomTTFonts = customfonts.BaseCustomTTFonts
+        In not lazy mode will force scan filesystem and sync
+        """
+        if lazy:
+            # lazy loading, scan only if no fonts in db
+            found_fonts_ids = self.search(cr, uid, [('path', '!=', '/dev/null')], context=context)
+            if not found_fonts_ids:
+                # no scan yet or no font found on the system, scan the filesystem
+                self._scan_disk(cr, uid, context=context)
+            elif len(customfonts.CustomTTFonts) == 0:
+                # CustomTTFonts list is empty
+                self._sync(cr, uid, context=context)
+        else:
+            self._scan_disk(cr, uid, context=context)
+        return True
 
-        found_fonts = {}
+    def _scan_disk(self, cr, uid, context=None):
+        """Scan the file system and register the result in database"""
+        found_fonts = []
         for font_path in customfonts.list_all_sysfonts():
             try:
                 font = ttfonts.TTFontFile(font_path)
                 _logger.debug("Found font %s at %s", font.name, font_path)
-                if not found_fonts.get(font.familyName):
+                found_fonts.append((font.familyName, font.name, font_path, font.styleName))
-                    found_fonts[font.familyName] = {'name': font.familyName}
-
-                mode = font.styleName.lower().replace(" ", "")
-
-                customfonts.CustomTTFonts.append((font.familyName, font.name, font_path, mode))
             except ttfonts.TTFError:
                 _logger.warning("Could not register Font %s", font_path)
 
-        # add default PDF fonts
+        for family, name, path, mode in found_fonts:
-        for family in customfonts.BasePDFFonts:
+            if not self.search(cr, uid, [('family', '=', family), ('name', '=', name)], context=context):
-            if not found_fonts.get(family):
+                self.create(cr, uid, {
-                found_fonts[family] = {'name': family}
+                    'family': family, 'name': name,
+                    'path': path, 'mode': mode,
+                }, context=context)
 
-        # remove deleted fonts
+        # remove fonts not present on the disk anymore
-        existing_font_ids = self.search(cr, uid, [], context=context)
+        existing_font_names = [name for (family, name, path, mode) in found_fonts]
-        existing_font_names = []
+        inexistant_fonts = self.search(cr, uid, [('name', 'not in', existing_font_names), ('path', '!=', '/dev/null')], context=context)
-        for font in self.browse(cr, uid, existing_font_ids):
+        if inexistant_fonts:
-            existing_font_names.append(font.name)
+            self.unlink(cr, uid, inexistant_fonts, context=context)
-            if font.name not in found_fonts.keys():
-                self.unlink(cr, uid, font.id, context=context)
 
-        # add unknown fonts
+        RegistryManager.signal_caches_change(cr.dbname)
-        for family, vals in found_fonts.items():
+        self._sync(cr, uid, context=context)
-            if family not in existing_font_names:
-                self.create(cr, uid, vals, context=context)
         return True
 
-    def init_no_scan(self, cr, uid, context=None):
+    def _sync(self, cr, uid, context=None):
-        """Add demo data for PDF fonts without scan (faster for db creation)"""
+        """Set the customfonts.CustomTTFonts list to the content of the database"""
-        for font in customfonts.BasePDFFonts:
+        customfonts.CustomTTFonts = []
-            if not self.search(cr, uid, [('name', '=', font)], context=context):
+        found_fonts_ids = self.search(cr, uid, [('path', '!=', '/dev/null')], context=context)
-                self.create(cr, uid, {'name':font}, context=context)
+        for font in self.browse(cr, uid, found_fonts_ids, context=None):
-        return True
+            customfonts.CustomTTFonts.append((font.family, font.name, font.path, font.mode))
+        return True
+
+    def clear_caches(self):
+        """Force worker to resync at next report loading by setting an empty font list"""
+        customfonts.CustomTTFonts = []
+        return super(res_font, self).clear_caches()

openerp/addons/base/security/ir.model.access.csv

@@ -43,6 +43,8 @@
 "access_ir_values_group_all","ir_values group_all","model_ir_values",,1,1,1,1
 "access_res_company_group_erp_manager","res_company group_erp_manager","model_res_company","group_erp_manager",1,1,1,1
 "access_res_company_group_user","res_company group_user","model_res_company",,1,0,0,0
+"access_res_font_group_erp_manager","res_font group_erp_manager","model_res_font","group_erp_manager",1,1,1,1
+"access_res_font_group_all","res_font group_all","model_res_font",,1,0,0,0
 "access_res_country_group_all","res_country group_user_all","model_res_country",,1,0,0,0
 "access_res_country_state_group_all","res_country_state group_user_all","model_res_country_state",,1,0,0,0
 "access_res_country_group_user","res_country group_user","model_res_country","group_partner_manager",1,1,1,1

openerp/report/interface.py

@@ -24,8 +24,6 @@ import re
 
 from lxml import etree
 
-import openerp
-
 import openerp
 import openerp.tools as tools
 import openerp.modules
@@ -33,6 +31,9 @@ import print_xml
 import render
 import urllib
 
+from openerp import SUPERUSER_ID
+from openerp.report.render.rml2pdf import customfonts
+
 #
 # coerce any type to a unicode string (to preserve non-ascii characters)
 # and escape XML entities
@@ -91,13 +92,16 @@ class report_rml(report_int):
         }
 
     def create(self, cr, uid, ids, datas, context):
+        registry = openerp.registry(cr.dbname)
         xml = self.create_xml(cr, uid, ids, datas, context)
         xml = tools.ustr(xml).encode('utf8')
         report_type = datas.get('report_type', 'pdf')
         if report_type == 'raw':
             return xml, report_type
+
+        registry['res.font'].font_scan(cr, SUPERUSER_ID, lazy=True, context=context)
+
         rml = self.create_rml(cr, xml, uid, context)
-        registry = openerp.registry(cr.dbname)
         ir_actions_report_xml_obj = registry['ir.actions.report.xml']
         report_xml_ids = ir_actions_report_xml_obj.search(cr, uid, [('report_name', '=', self.name[7:])], context=context)
         self.title = report_xml_ids and ir_actions_report_xml_obj.browse(cr,uid,report_xml_ids)[0].name or 'OpenERP Report'

openerp/report/render/rml2pdf/customfonts.py

@@ -21,10 +21,9 @@
 ##############################################################################
 
 from reportlab import rl_config
-from reportlab.lib import fonts
-from reportlab.pdfbase import pdfmetrics, ttfonts
 import logging
-import os,platform
+import glob
+import os
 
 # .apidoc title: TTF Font Table
 
@@ -39,35 +38,14 @@ and Ubuntu distros, we have to override the search path, too.
 """
 _logger = logging.getLogger(__name__)
 
-# Basic fonts family included in PDF standart, will always be in the font list
+CustomTTFonts = []
-BasePDFFonts = [
-    'Helvetica',
-    'Times',
-    'Courier'
-]
-
-# List of fonts found on the disk
-CustomTTFonts = BaseCustomTTFonts = [ ('Helvetica', "DejaVu Sans", "DejaVuSans.ttf", 'normal'),
-        ('Helvetica', "DejaVu Sans Bold", "DejaVuSans-Bold.ttf", 'bold'),
-        ('Helvetica', "DejaVu Sans Oblique", "DejaVuSans-Oblique.ttf", 'italic'),
-        ('Helvetica', "DejaVu Sans BoldOblique", "DejaVuSans-BoldOblique.ttf", 'bolditalic'),
-        ('Times', "Liberation Serif", "LiberationSerif-Regular.ttf", 'normal'),
-        ('Times', "Liberation Serif Bold", "LiberationSerif-Bold.ttf", 'bold'),
-        ('Times', "Liberation Serif Italic", "LiberationSerif-Italic.ttf", 'italic'),
-        ('Times', "Liberation Serif BoldItalic", "LiberationSerif-BoldItalic.ttf", 'bolditalic'),
-        ('Courier', "FreeMono", "FreeMono.ttf", 'normal'),
-        ('Courier', "FreeMono Bold", "FreeMonoBold.ttf", 'bold'),
-        ('Courier', "FreeMono Oblique", "FreeMonoOblique.ttf", 'italic'),
-        ('Courier', "FreeMono BoldOblique", "FreeMonoBoldOblique.ttf", 'bolditalic'),
-]
-
 
 # Search path for TTF files, in addition of rl_config.TTFSearchPath
 TTFSearchPath = [
             '/usr/share/fonts/truetype', # SuSE
             '/usr/share/fonts/dejavu', '/usr/share/fonts/liberation', # Fedora, RHEL
             '/usr/share/fonts/truetype/*','/usr/local/share/fonts' # Ubuntu,
-            '/usr/share/fonts/TTF/*', # at Mandriva/Mageia
+            '/usr/share/fonts/TTF/*', # Mandriva/Mageia
             '/usr/share/fonts/TTF', # Arch Linux
             '/usr/lib/openoffice/share/fonts/truetype/',
             '~/.fonts',
@@ -95,10 +73,8 @@ def list_all_sysfonts():
     # TTFOpenFile is not very good at it.
     searchpath = list(set(TTFSearchPath + rl_config.TTFSearchPath))
     for dirname in searchpath:
-        dirname = os.path.expanduser(dirname)
+        for filename in glob.glob(os.path.join(os.path.expanduser(dirname), '*.[Tt][Tt][Ff]')):
-        if os.path.exists(dirname):
+            filepath.append(filename)
-            for filename in [x for x in os.listdir(dirname) if x.lower().endswith('.ttf')]:
-                filepath.append(os.path.join(dirname, filename))
     return filepath
 
 def SetCustomFonts(rmldoc):
@@ -109,9 +85,9 @@ def SetCustomFonts(rmldoc):
         This function is called once per report, so it should
         avoid system-wide processing (cache it, instead).
     """
-    for name, font, filename, mode in CustomTTFonts:
+    for family, font, filename, mode in CustomTTFonts:
         if os.path.isabs(filename) and os.path.exists(filename):
-            rmldoc.setTTFontMapping(name, font, filename, mode)
+            rmldoc.setTTFontMapping(family, font, filename, mode)
     return True
 
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

openerp/report/render/rml2pdf/trml2pdf.py

@@ -56,8 +56,11 @@ def select_fontname(fontname, default_fontname):
         try:
             pdfmetrics.getFont(fontname)
         except Exception:
-            _logger.warning('Could not locate font %s, substituting default: %s',
+            addition = ""
-                fontname, default_fontname)
+            if " " in fontname:
+                addition = ". Your font contains spaces which is not valid in RML."
+            _logger.warning('Could not locate font %s, substituting default: %s%s',
+                fontname, default_fontname, addition)
             fontname = default_fontname
     return fontname
 
@@ -307,7 +310,7 @@ class _rml_doc(object):
             addMapping(face, 0, 1, fontname)    #italic
             addMapping(face, 1, 0, fontname)    #bold
             addMapping(face, 1, 1, fontname)    #italic and bold
-        elif (mode== 'normal') or (mode == 'regular'):
+        elif (mode== 'normal') or (mode == 'regular') or (mode == 'book'):
             addMapping(face, 0, 0, fontname)    #normal
         elif mode == 'italic':
             addMapping(face, 0, 1, fontname)    #italic

openerp/report/report_sxw.py

@@ -34,6 +34,7 @@ import zipfile
 import common
 
 import openerp
+from openerp import SUPERUSER_ID
 from openerp.osv.fields import float as float_field, function as function_field, datetime as datetime_field
 from openerp.tools.translate import _
 from openerp.tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT
@@ -419,10 +420,13 @@ class report_sxw(report_rml, preprocess.report):
             context = {}
         if self.internal_header:
             context.update(internal_header=self.internal_header)
+
         # skip osv.fields.sanitize_binary_value() because we want the raw bytes in all cases
         context.update(bin_raw=True)
         registry = openerp.registry(cr.dbname)
         ir_obj = registry['ir.actions.report.xml']
+        registry['res.font'].font_scan(cr, SUPERUSER_ID, lazy=True, context=context)
+
         report_xml_ids = ir_obj.search(cr, uid,
                 [('report_name', '=', self.name[7:])], context=context)
         if report_xml_ids: