diff --git a/bin/sql_db.py b/bin/sql_db.py index 17bfa340653..94336cde69a 100644 --- a/bin/sql_db.py +++ b/bin/sql_db.py @@ -63,7 +63,7 @@ re_into = re.compile('.* into "?([a-zA-Z_0-9]+)"? .*$'); sql_counter = 0 class Cursor(object): - IN_MAX = 1000 + IN_MAX = 1000 # decent limit on size of IN queries - guideline = Oracle limit __logger = logging.getLogger('db.cursor') def check(f): @@ -142,6 +142,12 @@ class Cursor(object): self.sql_into_log[res_into.group(1)][1] += delay return res + + def split_for_in_conditions(self, ids): + """Split a list of identifiers into one or more smaller tuples + safe for IN conditions, after uniquifying them.""" + return tools.misc.split_every(self.IN_MAX, set(ids)) + def print_log(self): global sql_counter sql_counter += self.sql_log_count diff --git a/bin/tools/misc.py b/bin/tools/misc.py index 284324873de..ff20966857e 100644 --- a/bin/tools/misc.py +++ b/bin/tools/misc.py @@ -32,6 +32,7 @@ import zipfile import release import socket import re +from itertools import islice if sys.version_info[:2] < (2, 4): from threadinglocal import local @@ -1267,6 +1268,18 @@ def detect_server_timezone(): return 'UTC' +def split_every(n, iterable, piece_maker=tuple): + """Splits an iterable into length-n pieces. The last piece will be shorter + if ``n`` does not evenly divide the iterable length. + @param ``piece_maker``: function to build the pieces + from the slices (tuple,list,...) + """ + iterator = iter(iterable) + piece = piece_maker(islice(iterator, n)) + while piece: + yield piece + piece = piece_maker(islice(iterator, n)) + if __name__ == '__main__': import doctest doctest.testmod()