[FIX] osv: Automatically retry the typical transaction serialization errors

lp bug: https://launchpad.net/bugs/992525 fixed

bzr revid: cto@openerp.com-20130213125255-ct0bf90pky2n6w3c

openerp/osv/osv.py

@@ -25,7 +25,7 @@ from functools import wraps
 import logging
 import threading
 
-from psycopg2 import IntegrityError, errorcodes
+from psycopg2 import IntegrityError, OperationalError, errorcodes
 
 import orm
 import openerp
@@ -36,8 +36,14 @@ from openerp.tools.translate import translate
 from openerp.osv.orm import MetaModel, Model, TransientModel, AbstractModel
 import openerp.exceptions
 
+import time
+import random
+
 _logger = logging.getLogger(__name__)
 
+PG_CONCURRENCY_ERRORS_TO_RETRY = (errorcodes.LOCK_NOT_AVAILABLE, errorcodes.SERIALIZATION_FAILURE, errorcodes.DEADLOCK_DETECTED)
+MAX_TRIES_ON_CONCURRENCY_FAILURE = 5
+
 # Deprecated.
 class except_osv(Exception):
     def __init__(self, name, value):
@@ -117,10 +123,21 @@ class object_proxy(object):
             def _(src):
                 return tr(src, 'code')
 
+            tries = 0
+            while True:
                 try:
                     if pooler.get_pool(dbname)._init:
                         raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
                     return f(self, dbname, *args, **kwargs)
+                except OperationalError, e:
+                    # Automatically retry the typical transaction serialization errors
+                    if not e.pgcode in PG_CONCURRENCY_ERRORS_TO_RETRY or tries >= MAX_TRIES_ON_CONCURRENCY_FAILURE:
+                        self.logger.warning("%s, maximum number of tries reached" % errorcodes.lookup(e.pgcode))
+                        raise
+                    wait_time = random.uniform(0.0, 2 ** tries)
+                    tries += 1
+                    self.logger.info("%s, retrying %d/%d in %.04f sec..." % (errorcodes.lookup(e.pgcode), tries, MAX_TRIES_ON_CONCURRENCY_FAILURE, wait_time))
+                    time.sleep(wait_time)
                 except orm.except_orm, inst:
                     raise except_osv(inst.name, inst.value)
                 except except_osv: