[ADD] Assets Bundle versioning and better cache handling

An asset bundle is now versionned with the dates of the
ir.ui.views that compose it and also with the dates of
the files and ir.attachments linked inside the bundle.

This new behavior is reflected in the bundle's lru cache
managment.

addons/web/controllers/main.py

@@ -49,6 +49,9 @@ else:
 env = jinja2.Environment(loader=loader, autoescape=True)
 env.filters["json"] = simplejson.dumps
 
+# 1 week cache for asset bundles as advised by Google Page Speed
+BUNDLE_MAXAGE = 60 * 60 * 24 * 7
+
 #----------------------------------------------------------
 # OpenERP Web helpers
 #----------------------------------------------------------
@@ -547,37 +550,31 @@ class Home(http.Controller):
     def login(self, db, login, key, redirect="/web", **kw):
         return login_and_redirect(db, login, key, redirect_url=redirect)
 
-    @http.route('/web/js/<xmlid>', type='http', auth="public")
+    @http.route([
-    def js_bundle(self, xmlid, **kw):
+        '/web/js/<xmlid>',
-        # manifest backward compatible mode, to be removed
+        '/web/js/<xmlid>/<sha>',
-        values = {'manifest_list': manifest_list}
+    ], type='http', auth='public')
+    def js_bundle(self, xmlid, sha=None, **kw):
         try:
-            assets_html = request.render(xmlid, lazy=False, qcontext=values)
+            bundle = AssetsBundle(xmlid, debug=request.debug)
         except QWebTemplateNotFound:
             return request.not_found()
-        bundle = AssetsBundle(xmlid, assets_html, debug=request.debug)
 
-        response = request.make_response(
+        response = request.make_response(bundle.js(), [('Content-Type', 'application/javascript')])
-            bundle.js(), [('Content-Type', 'application/javascript')])
+        return make_conditional(response, bundle.last_modified, max_age=BUNDLE_MAXAGE)
 
-        # TODO: check that we don't do weird lazy overriding of __call__ which break body-removal
+    @http.route([
-        return make_conditional(
+        '/web/css/<xmlid>',
-            response, bundle.last_modified, bundle.checksum, max_age=60*60*24)
+        '/web/css/<xmlid>/<sha>',
-
+    ], type='http', auth='public')
-    @http.route('/web/css/<xmlid>', type='http', auth='public')
+    def css_bundle(self, xmlid, sha=None, **kw):
-    def css_bundle(self, xmlid, **kw):
-        values = {'manifest_list': manifest_list} # manifest backward compatible mode, to be removed
         try:
-            assets_html = request.render(xmlid, lazy=False, qcontext=values)
+            bundle = AssetsBundle(xmlid, debug=request.debug)
         except QWebTemplateNotFound:
             return request.not_found()
-        bundle = AssetsBundle(xmlid, assets_html, debug=request.debug)
 
-        response = request.make_response(
+        response = request.make_response(bundle.css(), [('Content-Type', 'text/css')])
-            bundle.css(), [('Content-Type', 'text/css')])
+        return make_conditional(response, bundle.last_modified, max_age=BUNDLE_MAXAGE)
-
-        return make_conditional(
-            response, bundle.last_modified, bundle.checksum, max_age=60*60*24)
 
 class WebClient(http.Controller):
 

openerp/addons/base/ir/ir_qweb.py

@@ -414,16 +414,15 @@ class QWeb(orm.AbstractModel):
 
     def render_tag_call_assets(self, element, template_attributes, generated_attributes, qwebcontext):
         """ This special 't-call' tag can be used in order to aggregate/minify javascript and css assets"""
-        name = template_attributes['call-assets']
+        if element.childNodes:
-
+            # An asset bundle is rendered in two differents contexts (when genereting html and
-        # Backward compatibility hack for manifest usage
+            # when generating the bundle itself) so they must be qwebcontext free
-        qwebcontext['manifest_list'] = openerp.addons.web.controllers.main.manifest_list
+            # even '0' variable is forbidden
-
+            template = qwebcontext.get('__template__')
-        d = qwebcontext.copy()
+            raise QWebException("t-call-assets cannot contain children nodes", template=template)
-        d.context['inherit_branding'] = False
+        xmlid = template_attributes['call-assets']
-        content = self.render_tag_call(
+        cr, uid, context = [getattr(qwebcontext, attr) for attr in ('cr', 'uid', 'context')]
-            element, {'call': name}, generated_attributes, d)
+        bundle = AssetsBundle(xmlid, cr=cr, uid=uid, context=context, registry=self.pool)
-        bundle = AssetsBundle(name, html=content)
         css = self.get_attr_bool(template_attributes.get('css'), default=True)
         js = self.get_attr_bool(template_attributes.get('js'), default=True)
         return bundle.to_html(css=css, js=js, debug=bool(qwebcontext.get('debug')))
@@ -1004,13 +1003,32 @@ class AssetsBundle(object):
     cache = openerp.tools.lru.LRU(32)
     rx_css_import = re.compile("(@import[^;{]+;?)", re.M)
 
-    def __init__(self, xmlid, html=None, debug=False):
+    def __init__(self, xmlid, debug=False, cr=None, uid=None, context=None, registry=None):
-        self.debug = debug
         self.xmlid = xmlid
+        self.debug = debug
+        self.cr = request.cr if cr is None else cr
+        self.uid = request.uid if uid is None else uid
+        self.context = request.context if context is None else context
+        self.registry = request.registry if registry is None else registry
         self.javascripts = []
         self.stylesheets = []
         self.remains = []
         self._checksum = None
+        self._last_modified = datetime.datetime(1970, 1, 1)
+
+        last_updates = []
+        context = self.context.copy()
+        context.update(
+            inherit_branding=False,
+            collect_last_updates=last_updates,
+        )
+        html = self.registry['ir.ui.view'].render(self.cr, self.uid, xmlid, context=context)
+        if last_updates:
+            # ir.ui.view are orm cached. If the bundle view is actually cached, we won't receive
+            # last_updates. In this case we consider it's already cached in self.cache and thus
+            # only the dates of the files/ir.attachements assets will be compared.
+            server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT
+            self._last_modified = datetime.datetime.strptime(max(last_updates), server_format)
         if html:
             self.parse(html)
 
@@ -1055,9 +1073,9 @@ class AssetsBundle(object):
                     response.append(jscript.to_html())
         else:
             if css and self.stylesheets:
-                response.append('<link href="/web/css/%s" rel="stylesheet"/>' % self.xmlid)
+                response.append('<link href="/web/css/%s/%s" rel="stylesheet"/>' % (self.xmlid, self.version))
             if js and self.javascripts:
-                response.append('<script type="text/javascript" src="/web/js/%s"></script>' % self.xmlid)
+                response.append('<script type="text/javascript" src="/web/js/%s/%s"></script>' % (self.xmlid, self.version))
         response.extend(self.remains)
         return sep + sep.join(response)
 
@@ -1066,9 +1084,13 @@ class AssetsBundle(object):
         return max(itertools.chain(
             (asset.last_modified for asset in self.javascripts),
             (asset.last_modified for asset in self.stylesheets),
-            [datetime.datetime(1970, 1, 1)],
+            [self._last_modified],
         ))
 
+    @lazy_property
+    def version(self):
+        return hashlib.sha1(str(self.last_modified)).hexdigest()[0:7]
+
     @lazy_property
     def checksum(self):
         checksum = hashlib.new('sha1')
@@ -1077,17 +1099,23 @@ class AssetsBundle(object):
         return checksum.hexdigest()
 
     def js(self):
-        key = 'js_' + self.checksum
+        key = 'js_%s' % self.xmlid
+        if key in self.cache and self.cache[key][0] != self.version:
+            # Invalidate cache on version mismach
+            self.cache.pop(key)
         if key not in self.cache:
             content =';\n'.join(asset.minify() for asset in self.javascripts)
-            self.cache[key] = content
+            self.cache[key] = (self.version, content)
         if self.debug:
             return "/*\n%s\n*/\n" % '\n'.join(
                 [asset.url for asset in self.javascripts if asset.url]) + self.cache[key]
-        return self.cache[key]
+        return self.cache[key][1]
 
     def css(self):
-        key = 'css_' + self.checksum
+        key = 'css_%s' % self.xmlid
+        if key in self.cache and self.cache[key][0] != self.version:
+            # Invalidate cache on version mismach
+            self.cache.pop(key)
         if key not in self.cache:
             content = '\n'.join(asset.minify() for asset in self.stylesheets)
             # move up all @import rules to the top
@@ -1100,11 +1128,11 @@ class AssetsBundle(object):
 
             matches.append(content)
             content = u'\n'.join(matches)
-            self.cache[key] = content
+            self.cache[key] = (self.version, content)
         if self.debug:
             return "/*\n%s\n*/\n" % '\n'.join(
                 [asset.url for asset in self.javascripts if asset.url]) + self.cache[key]
-        return self.cache[key]
+        return self.cache[key][1]
 
 class WebAsset(object):
     def __init__(self, bundle, inline=None, url=None, cr=None, uid=SUPERUSER_ID):
@@ -1116,7 +1144,7 @@ class WebAsset(object):
         self._filename = None
         self._ir_attach = None
         if not inline and not url:
-            raise Exception("A bundle should either be inlined or url linked")
+            raise Exception("An asset should either be inlined or url linked")
 
     def stat(self):
         if not (self.inline or self._filename or self._ir_attach):
@@ -1128,8 +1156,10 @@ class WebAsset(object):
             except Exception:
                 try:
                     # Test url against ir.attachments
+                    fields = ['__last_update', 'datas', 'mimetype']
                     domain = [('type', '=', 'binary'), ('url', '=', self.url)]
-                    attach = request.registry['ir.attachment'].search_read(self.cr, self.uid, domain, ['__last_update', 'datas', 'mimetype'], context=request.context)
+                    ira = request.registry['ir.attachment']
+                    attach = ira.search_read(self.cr, self.uid, domain, fields, context=request.context)
                     self._ir_attach = attach[0]
                 except Exception:
                     raise AssetNotFound(url=self.url)

openerp/addons/base/ir/ir_ui_view.py

@@ -367,9 +367,13 @@ class view(osv.osv):
             ])
         view_ids = self.search(cr, uid, conditions, context=context)
 
-        return [(view.arch, view.id)
+        inheriting_views = []
-                for view in self.browse(cr, 1, view_ids, context)
+        for view in self.browse(cr, 1, view_ids, context):
-                if not (view.groups_id and user_groups.isdisjoint(view.groups_id))]
+            if not (view.groups_id and user_groups.isdisjoint(view.groups_id)):
+                inheriting_views.append((view.arch, view.id))
+                if 'collect_last_updates' in context:
+                    context['collect_last_updates'].append(view.write_date)
+        return inheriting_views
 
     def raise_view_error(self, cr, uid, message, view_id, context=None):
         view = self.browse(cr, uid, view_id, context)
@@ -519,7 +523,7 @@ class view(osv.osv):
         if context is None: context = {}
         if root_id is None:
             root_id = source_id
-        sql_inherit = self.pool['ir.ui.view'].get_inheriting_views_arch(cr, uid, source_id, model, context=context)
+        sql_inherit = self.get_inheriting_views_arch(cr, uid, source_id, model, context=context)
         for (specs, view_id) in sql_inherit:
             specs_tree = etree.fromstring(specs.encode('utf-8'))
             if context.get('inherit_branding'):
@@ -546,13 +550,18 @@ class view(osv.osv):
             v = v.inherit_id
         root_id = v.id
 
+        collect_last_updates = 'collect_last_updates' in context
         # arch and model fields are always returned
         if fields:
             fields = list(set(fields) | set(['arch', 'model']))
+            if collect_last_updates and 'write_date' not in fields:
+                fields.append('write_date')
 
         # read the view arch
         [view] = self.read(cr, uid, [root_id], fields=fields, context=context)
         view_arch = etree.fromstring(view['arch'].encode('utf-8'))
+        if collect_last_updates:
+            context['collect_last_updates'].append(view['write_date'])
         if not v.inherit_id:
             arch_tree = view_arch
         else: