From aa97aaa9d7ca7adc7ed46ee3f4805d30f5717b4e Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Tue, 24 Jun 2014 18:52:38 +0200 Subject: [PATCH] [ADD] Assets Bundle versioning and better cache handling An asset bundle is now versionned with the dates of the ir.ui.views that compose it and also with the dates of the files and ir.attachments linked inside the bundle. This new behavior is reflected in the bundle's lru cache managment. --- addons/web/controllers/main.py | 41 +++++++-------- openerp/addons/base/ir/ir_qweb.py | 76 +++++++++++++++++++--------- openerp/addons/base/ir/ir_ui_view.py | 17 +++++-- 3 files changed, 85 insertions(+), 49 deletions(-) diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 98f65075922..09416179725 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -49,6 +49,9 @@ else: env = jinja2.Environment(loader=loader, autoescape=True) env.filters["json"] = simplejson.dumps +# 1 week cache for asset bundles as advised by Google Page Speed +BUNDLE_MAXAGE = 60 * 60 * 24 * 7 + #---------------------------------------------------------- # OpenERP Web helpers #---------------------------------------------------------- @@ -547,37 +550,31 @@ class Home(http.Controller): def login(self, db, login, key, redirect="/web", **kw): return login_and_redirect(db, login, key, redirect_url=redirect) - @http.route('/web/js/', type='http', auth="public") - def js_bundle(self, xmlid, **kw): - # manifest backward compatible mode, to be removed - values = {'manifest_list': manifest_list} + @http.route([ + '/web/js/', + '/web/js//', + ], type='http', auth='public') + def js_bundle(self, xmlid, sha=None, **kw): try: - assets_html = request.render(xmlid, lazy=False, qcontext=values) + bundle = AssetsBundle(xmlid, debug=request.debug) except QWebTemplateNotFound: return request.not_found() - bundle = AssetsBundle(xmlid, assets_html, debug=request.debug) - response = request.make_response( - bundle.js(), [('Content-Type', 'application/javascript')]) + response = request.make_response(bundle.js(), [('Content-Type', 'application/javascript')]) + return make_conditional(response, bundle.last_modified, max_age=BUNDLE_MAXAGE) - # TODO: check that we don't do weird lazy overriding of __call__ which break body-removal - return make_conditional( - response, bundle.last_modified, bundle.checksum, max_age=60*60*24) - - @http.route('/web/css/', type='http', auth='public') - def css_bundle(self, xmlid, **kw): - values = {'manifest_list': manifest_list} # manifest backward compatible mode, to be removed + @http.route([ + '/web/css/', + '/web/css//', + ], type='http', auth='public') + def css_bundle(self, xmlid, sha=None, **kw): try: - assets_html = request.render(xmlid, lazy=False, qcontext=values) + bundle = AssetsBundle(xmlid, debug=request.debug) except QWebTemplateNotFound: return request.not_found() - bundle = AssetsBundle(xmlid, assets_html, debug=request.debug) - response = request.make_response( - bundle.css(), [('Content-Type', 'text/css')]) - - return make_conditional( - response, bundle.last_modified, bundle.checksum, max_age=60*60*24) + response = request.make_response(bundle.css(), [('Content-Type', 'text/css')]) + return make_conditional(response, bundle.last_modified, max_age=BUNDLE_MAXAGE) class WebClient(http.Controller): diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index d51d7cc79c6..87ba351cb9a 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -414,16 +414,15 @@ class QWeb(orm.AbstractModel): def render_tag_call_assets(self, element, template_attributes, generated_attributes, qwebcontext): """ This special 't-call' tag can be used in order to aggregate/minify javascript and css assets""" - name = template_attributes['call-assets'] - - # Backward compatibility hack for manifest usage - qwebcontext['manifest_list'] = openerp.addons.web.controllers.main.manifest_list - - d = qwebcontext.copy() - d.context['inherit_branding'] = False - content = self.render_tag_call( - element, {'call': name}, generated_attributes, d) - bundle = AssetsBundle(name, html=content) + if element.childNodes: + # An asset bundle is rendered in two differents contexts (when genereting html and + # when generating the bundle itself) so they must be qwebcontext free + # even '0' variable is forbidden + template = qwebcontext.get('__template__') + raise QWebException("t-call-assets cannot contain children nodes", template=template) + xmlid = template_attributes['call-assets'] + cr, uid, context = [getattr(qwebcontext, attr) for attr in ('cr', 'uid', 'context')] + bundle = AssetsBundle(xmlid, cr=cr, uid=uid, context=context, registry=self.pool) css = self.get_attr_bool(template_attributes.get('css'), default=True) js = self.get_attr_bool(template_attributes.get('js'), default=True) return bundle.to_html(css=css, js=js, debug=bool(qwebcontext.get('debug'))) @@ -1004,13 +1003,32 @@ class AssetsBundle(object): cache = openerp.tools.lru.LRU(32) rx_css_import = re.compile("(@import[^;{]+;?)", re.M) - def __init__(self, xmlid, html=None, debug=False): - self.debug = debug + def __init__(self, xmlid, debug=False, cr=None, uid=None, context=None, registry=None): self.xmlid = xmlid + self.debug = debug + self.cr = request.cr if cr is None else cr + self.uid = request.uid if uid is None else uid + self.context = request.context if context is None else context + self.registry = request.registry if registry is None else registry self.javascripts = [] self.stylesheets = [] self.remains = [] self._checksum = None + self._last_modified = datetime.datetime(1970, 1, 1) + + last_updates = [] + context = self.context.copy() + context.update( + inherit_branding=False, + collect_last_updates=last_updates, + ) + html = self.registry['ir.ui.view'].render(self.cr, self.uid, xmlid, context=context) + if last_updates: + # ir.ui.view are orm cached. If the bundle view is actually cached, we won't receive + # last_updates. In this case we consider it's already cached in self.cache and thus + # only the dates of the files/ir.attachements assets will be compared. + server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT + self._last_modified = datetime.datetime.strptime(max(last_updates), server_format) if html: self.parse(html) @@ -1055,9 +1073,9 @@ class AssetsBundle(object): response.append(jscript.to_html()) else: if css and self.stylesheets: - response.append('' % self.xmlid) + response.append('' % (self.xmlid, self.version)) if js and self.javascripts: - response.append('' % self.xmlid) + response.append('' % (self.xmlid, self.version)) response.extend(self.remains) return sep + sep.join(response) @@ -1066,9 +1084,13 @@ class AssetsBundle(object): return max(itertools.chain( (asset.last_modified for asset in self.javascripts), (asset.last_modified for asset in self.stylesheets), - [datetime.datetime(1970, 1, 1)], + [self._last_modified], )) + @lazy_property + def version(self): + return hashlib.sha1(str(self.last_modified)).hexdigest()[0:7] + @lazy_property def checksum(self): checksum = hashlib.new('sha1') @@ -1077,17 +1099,23 @@ class AssetsBundle(object): return checksum.hexdigest() def js(self): - key = 'js_' + self.checksum + key = 'js_%s' % self.xmlid + if key in self.cache and self.cache[key][0] != self.version: + # Invalidate cache on version mismach + self.cache.pop(key) if key not in self.cache: content =';\n'.join(asset.minify() for asset in self.javascripts) - self.cache[key] = content + self.cache[key] = (self.version, content) if self.debug: return "/*\n%s\n*/\n" % '\n'.join( [asset.url for asset in self.javascripts if asset.url]) + self.cache[key] - return self.cache[key] + return self.cache[key][1] def css(self): - key = 'css_' + self.checksum + key = 'css_%s' % self.xmlid + if key in self.cache and self.cache[key][0] != self.version: + # Invalidate cache on version mismach + self.cache.pop(key) if key not in self.cache: content = '\n'.join(asset.minify() for asset in self.stylesheets) # move up all @import rules to the top @@ -1100,11 +1128,11 @@ class AssetsBundle(object): matches.append(content) content = u'\n'.join(matches) - self.cache[key] = content + self.cache[key] = (self.version, content) if self.debug: return "/*\n%s\n*/\n" % '\n'.join( [asset.url for asset in self.javascripts if asset.url]) + self.cache[key] - return self.cache[key] + return self.cache[key][1] class WebAsset(object): def __init__(self, bundle, inline=None, url=None, cr=None, uid=SUPERUSER_ID): @@ -1116,7 +1144,7 @@ class WebAsset(object): self._filename = None self._ir_attach = None if not inline and not url: - raise Exception("A bundle should either be inlined or url linked") + raise Exception("An asset should either be inlined or url linked") def stat(self): if not (self.inline or self._filename or self._ir_attach): @@ -1128,8 +1156,10 @@ class WebAsset(object): except Exception: try: # Test url against ir.attachments + fields = ['__last_update', 'datas', 'mimetype'] domain = [('type', '=', 'binary'), ('url', '=', self.url)] - attach = request.registry['ir.attachment'].search_read(self.cr, self.uid, domain, ['__last_update', 'datas', 'mimetype'], context=request.context) + ira = request.registry['ir.attachment'] + attach = ira.search_read(self.cr, self.uid, domain, fields, context=request.context) self._ir_attach = attach[0] except Exception: raise AssetNotFound(url=self.url) diff --git a/openerp/addons/base/ir/ir_ui_view.py b/openerp/addons/base/ir/ir_ui_view.py index 4018d779e53..b6fed6ddfbd 100644 --- a/openerp/addons/base/ir/ir_ui_view.py +++ b/openerp/addons/base/ir/ir_ui_view.py @@ -367,9 +367,13 @@ class view(osv.osv): ]) view_ids = self.search(cr, uid, conditions, context=context) - return [(view.arch, view.id) - for view in self.browse(cr, 1, view_ids, context) - if not (view.groups_id and user_groups.isdisjoint(view.groups_id))] + inheriting_views = [] + for view in self.browse(cr, 1, view_ids, context): + if not (view.groups_id and user_groups.isdisjoint(view.groups_id)): + inheriting_views.append((view.arch, view.id)) + if 'collect_last_updates' in context: + context['collect_last_updates'].append(view.write_date) + return inheriting_views def raise_view_error(self, cr, uid, message, view_id, context=None): view = self.browse(cr, uid, view_id, context) @@ -519,7 +523,7 @@ class view(osv.osv): if context is None: context = {} if root_id is None: root_id = source_id - sql_inherit = self.pool['ir.ui.view'].get_inheriting_views_arch(cr, uid, source_id, model, context=context) + sql_inherit = self.get_inheriting_views_arch(cr, uid, source_id, model, context=context) for (specs, view_id) in sql_inherit: specs_tree = etree.fromstring(specs.encode('utf-8')) if context.get('inherit_branding'): @@ -546,13 +550,18 @@ class view(osv.osv): v = v.inherit_id root_id = v.id + collect_last_updates = 'collect_last_updates' in context # arch and model fields are always returned if fields: fields = list(set(fields) | set(['arch', 'model'])) + if collect_last_updates and 'write_date' not in fields: + fields.append('write_date') # read the view arch [view] = self.read(cr, uid, [root_id], fields=fields, context=context) view_arch = etree.fromstring(view['arch'].encode('utf-8')) + if collect_last_updates: + context['collect_last_updates'].append(view['write_date']) if not v.inherit_id: arch_tree = view_arch else: