From 9fbdd2d83df002e2cba9b02a96cbd43d2996d42d Mon Sep 17 00:00:00 2001 From: Christophe Matthieu Date: Wed, 11 Feb 2015 18:13:49 +0100 Subject: [PATCH] [IMP] website: improve speed of test crawl and load only one slug by route If they are this routes: /partner/p-1 /partner/p-2 ... /partner/grade-1/p-1 /partner/grade-1/p-2 ... /partner/grade-2/p-1 /partner/grade-2/p-2 ... We want test only one time the routes: /partner/p-1 /partner/grade-1/p-1 --- addons/website/tests/test_crawl.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/addons/website/tests/test_crawl.py b/addons/website/tests/test_crawl.py index 56cd6f181ca..a548cb05af0 100644 --- a/addons/website/tests/test_crawl.py +++ b/addons/website/tests/test_crawl.py @@ -1,15 +1,12 @@ # -*- coding: utf-8 -*- import logging import urlparse -import unittest2 -import urllib2 import time -import werkzeug.urls import lxml.html import openerp -from openerp import tools +import re _logger = logging.getLogger(__name__) @@ -25,12 +22,15 @@ class Crawler(openerp.tests.HttpCase): post_install = True def crawl(self, url, seen=None, msg=''): - if seen == None: + if seen == None: seen = set() - if url in seen: + + url_slug = re.sub(r"[/](([^/=?&]+-)?[0-9]+)([/]|$)", '//', url) + url_slug = re.sub(r"([^/=?&]+)=[^/=?&]+", '\g<1>=param', url_slug) + if url_slug in seen: return seen else: - seen.add(url) + seen.add(url_slug) _logger.info("%s %s", msg, url) r = self.url_open(url)