[IMP] refactoring of readgroup method. It should have the same functionality, but is much simpler. This is necessary to implement eager groupby (orm.py)
bzr revid: ged@openerp.com-20140324085046-zpfpcostivf8454q
This commit is contained in:
parent
31645ea1cf
commit
0eb9944222
|
@ -2222,7 +2222,7 @@ class BaseModel(object):
|
||||||
self._name, order_part)
|
self._name, order_part)
|
||||||
return groupby_terms, orderby_terms
|
return groupby_terms, orderby_terms
|
||||||
|
|
||||||
def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context=None, orderby=False):
|
def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False):
|
||||||
"""
|
"""
|
||||||
Get the list of records in list view grouped by the given ``groupby`` fields
|
Get the list of records in list view grouped by the given ``groupby`` fields
|
||||||
|
|
||||||
|
@ -2252,91 +2252,80 @@ class BaseModel(object):
|
||||||
* if user tries to bypass access rules for read on the requested object
|
* if user tries to bypass access rules for read on the requested object
|
||||||
|
|
||||||
"""
|
"""
|
||||||
context = context or {}
|
|
||||||
self.check_access_rights(cr, uid, 'read')
|
self.check_access_rights(cr, uid, 'read')
|
||||||
if not fields:
|
|
||||||
fields = self._columns.keys()
|
# Step 0 : preparing some useful variables
|
||||||
|
fields = fields or self._columns.keys()
|
||||||
|
fget = self.fields_get(cr, uid, fields)
|
||||||
|
if isinstance(groupby, basestring):
|
||||||
|
groupby = [groupby]
|
||||||
|
split_groupby = groupby[0].split(':') if groupby else None
|
||||||
|
first_groupby = split_groupby[0] if split_groupby else None
|
||||||
|
groupby_function = split_groupby[1] if split_groupby and len(split_groupby) == 2 else None
|
||||||
|
interval = groupby_function if groupby_function else 'month'
|
||||||
|
groupby_type = fget[first_groupby]['type'] if first_groupby else None
|
||||||
|
if groupby_type in ('date', 'datetime'):
|
||||||
|
dt_format = DEFAULT_SERVER_DATETIME_FORMAT if groupby_type == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
|
||||||
|
tz_convert = groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones
|
||||||
|
time_display_format = {
|
||||||
|
'day': 'dd MMM YYYY',
|
||||||
|
'week': "'W'w YYYY",
|
||||||
|
'month': 'MMMM YYYY',
|
||||||
|
'quarter': 'QQQ YYYY',
|
||||||
|
'year': 'YYYY'}[interval]
|
||||||
|
time_interval = {
|
||||||
|
'day': dateutil.relativedelta.relativedelta(months=3),
|
||||||
|
'week': datetime.timedelta(days=7),
|
||||||
|
'month': dateutil.relativedelta.relativedelta(months=1),
|
||||||
|
'quarter': dateutil.relativedelta.relativedelta(months=3),
|
||||||
|
'year': dateutil.relativedelta.relativedelta(years=1)}[interval]
|
||||||
|
|
||||||
query = self._where_calc(cr, uid, domain, context=context)
|
query = self._where_calc(cr, uid, domain, context=context)
|
||||||
|
|
||||||
|
# Step 1: security stuff
|
||||||
|
# add relevant ir_rules to the where clause, perform some basic sanity checks
|
||||||
self._apply_ir_rules(cr, uid, query, 'read', context=context)
|
self._apply_ir_rules(cr, uid, query, 'read', context=context)
|
||||||
|
if first_groupby:
|
||||||
# Take care of adding join(s) if groupby is an '_inherits'ed field
|
assert first_groupby in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
|
||||||
groupby_list = groupby
|
groupby_def = self._columns.get(first_groupby) or (self._inherit_fields.get(first_groupby) and self._inherit_fields.get(first_groupby)[2])
|
||||||
qualified_groupby_field = groupby
|
|
||||||
if groupby:
|
|
||||||
if isinstance(groupby, list):
|
|
||||||
groupby = groupby[0]
|
|
||||||
splitted_groupby = groupby.split(':')
|
|
||||||
if len(splitted_groupby) == 2:
|
|
||||||
groupby = splitted_groupby[0]
|
|
||||||
groupby_function = splitted_groupby[1]
|
|
||||||
else:
|
|
||||||
groupby_function = False
|
|
||||||
qualified_groupby_field = self._inherits_join_calc(groupby, query)
|
|
||||||
|
|
||||||
if groupby:
|
|
||||||
assert not groupby or groupby in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
|
|
||||||
groupby_def = self._columns.get(groupby) or (self._inherit_fields.get(groupby) and self._inherit_fields.get(groupby)[2])
|
|
||||||
assert groupby_def and groupby_def._classic_write, "Fields in 'groupby' must be regular database-persisted fields (no function or related fields), or function fields with store=True"
|
assert groupby_def and groupby_def._classic_write, "Fields in 'groupby' must be regular database-persisted fields (no function or related fields), or function fields with store=True"
|
||||||
|
if not (first_groupby in fget):
|
||||||
|
# Don't allow arbitrary values, as this would be a SQL injection vector!
|
||||||
|
raise except_orm(_('Invalid group_by'),
|
||||||
|
_('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(first_groupby,))
|
||||||
|
|
||||||
|
# Step 2: preparing the query:
|
||||||
|
# compute aggregated fields, format groupbys, adjust timezone, ...
|
||||||
|
aggregated_fields = [
|
||||||
|
f for f in fields
|
||||||
|
if f not in ('id', 'sequence', groupby)
|
||||||
|
if fget[f]['type'] in ('integer', 'float')
|
||||||
|
if (f in self._all_columns and getattr(self._all_columns[f].column, '_classic_write'))]
|
||||||
|
|
||||||
|
field_formatter = lambda f: (fget[f].get('group_operator', 'sum'), self._inherits_join_calc(f, query), f)
|
||||||
|
select_terms = ["%s(%s) AS %s" % field_formatter(f) for f in aggregated_fields]
|
||||||
|
|
||||||
|
qualified_groupby_field = self._inherits_join_calc(first_groupby, query) if first_groupby else None
|
||||||
|
|
||||||
# TODO it seems fields_get can be replaced by _all_columns (no need for translation)
|
|
||||||
fget = self.fields_get(cr, uid, fields)
|
|
||||||
group_by_params = {}
|
|
||||||
select_terms = []
|
|
||||||
groupby_type = None
|
|
||||||
if groupby:
|
|
||||||
if fget.get(groupby):
|
|
||||||
groupby_type = fget[groupby]['type']
|
|
||||||
if groupby_type in ('date', 'datetime'):
|
if groupby_type in ('date', 'datetime'):
|
||||||
if groupby_function:
|
if tz_convert:
|
||||||
interval = groupby_function
|
|
||||||
else:
|
|
||||||
interval = 'month'
|
|
||||||
|
|
||||||
if interval == 'day':
|
|
||||||
display_format = 'dd MMM YYYY'
|
|
||||||
elif interval == 'week':
|
|
||||||
display_format = "'W'w YYYY"
|
|
||||||
elif interval == 'month':
|
|
||||||
display_format = 'MMMM YYYY'
|
|
||||||
elif interval == 'quarter':
|
|
||||||
display_format = 'QQQ YYYY'
|
|
||||||
elif interval == 'year':
|
|
||||||
display_format = 'YYYY'
|
|
||||||
|
|
||||||
if groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones:
|
|
||||||
# Convert groupby result to user TZ to avoid confusion!
|
# Convert groupby result to user TZ to avoid confusion!
|
||||||
# PostgreSQL is compatible with all pytz timezone names, so we can use them
|
# PostgreSQL is compatible with all pytz timezone names, so we can use them
|
||||||
# directly for conversion, starting with timestamps stored in UTC.
|
# directly for conversion, starting with timestamps stored in UTC.
|
||||||
timezone = context.get('tz', 'UTC')
|
timezone = context.get('tz', 'UTC')
|
||||||
qualified_groupby_field = "timezone('%s', timezone('UTC',%s))" % (timezone, qualified_groupby_field)
|
qualified_groupby_field = "timezone('%s', timezone('UTC',%s))" % (timezone, qualified_groupby_field)
|
||||||
qualified_groupby_field = "date_trunc('%s', %s)" % (interval, qualified_groupby_field)
|
qualified_groupby_field = "date_trunc('%s', %s)" % (interval, qualified_groupby_field)
|
||||||
elif groupby_type == 'boolean':
|
|
||||||
|
if groupby_type == 'boolean':
|
||||||
qualified_groupby_field = "coalesce(%s,false)" % qualified_groupby_field
|
qualified_groupby_field = "coalesce(%s,false)" % qualified_groupby_field
|
||||||
select_terms.append("%s as %s " % (qualified_groupby_field, groupby))
|
|
||||||
else:
|
|
||||||
# Don't allow arbitrary values, as this would be a SQL injection vector!
|
|
||||||
raise except_orm(_('Invalid group_by'),
|
|
||||||
_('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(groupby,))
|
|
||||||
|
|
||||||
aggregated_fields = [
|
if first_groupby:
|
||||||
f for f in fields
|
select_terms.append("%s as %s " % (qualified_groupby_field, first_groupby))
|
||||||
if f not in ('id', 'sequence', groupby)
|
|
||||||
if fget[f]['type'] in ('integer', 'float')
|
|
||||||
if (f in self._all_columns and getattr(self._all_columns[f].column, '_classic_write'))]
|
|
||||||
for f in aggregated_fields:
|
|
||||||
group_operator = fget[f].get('group_operator', 'sum')
|
|
||||||
qualified_field = self._inherits_join_calc(f, query)
|
|
||||||
select_terms.append("%s(%s) AS %s" % (group_operator, qualified_field, f))
|
|
||||||
|
|
||||||
order = orderby or groupby or ''
|
order = orderby or first_groupby or ''
|
||||||
groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, groupby, qualified_groupby_field, query, groupby_type)
|
groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, first_groupby, qualified_groupby_field, query, groupby_type)
|
||||||
|
|
||||||
from_clause, where_clause, where_clause_params = query.get_sql()
|
from_clause, where_clause, where_clause_params = query.get_sql()
|
||||||
if len(groupby_list) < 2 and context.get('group_by_no_leaf'):
|
|
||||||
count_field = '_'
|
|
||||||
else:
|
|
||||||
count_field = groupby
|
|
||||||
|
|
||||||
prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
|
prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
|
||||||
prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''
|
prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''
|
||||||
|
@ -2352,7 +2341,7 @@ class BaseModel(object):
|
||||||
%(offset)s
|
%(offset)s
|
||||||
""" % {
|
""" % {
|
||||||
'table': self._table,
|
'table': self._table,
|
||||||
'count_field': count_field,
|
'count_field': '_' if (len(groupby) < 2 and context.get('group_by_no_leaf')) else first_groupby,
|
||||||
'extra_fields': prefix_terms(',', select_terms),
|
'extra_fields': prefix_terms(',', select_terms),
|
||||||
'from': from_clause,
|
'from': from_clause,
|
||||||
'where': prefix_term('WHERE', where_clause),
|
'where': prefix_term('WHERE', where_clause),
|
||||||
|
@ -2362,73 +2351,54 @@ class BaseModel(object):
|
||||||
'offset': prefix_term('OFFSET', int(offset) if limit else None),
|
'offset': prefix_term('OFFSET', int(offset) if limit else None),
|
||||||
}
|
}
|
||||||
cr.execute(query, where_clause_params)
|
cr.execute(query, where_clause_params)
|
||||||
alldata = {}
|
|
||||||
fetched_data = cr.dictfetchall()
|
|
||||||
|
|
||||||
data_ids = []
|
if not first_groupby:
|
||||||
for r in fetched_data:
|
return {r.pop('id'): r for r in cr.dictfetchall() }
|
||||||
for fld, val in r.items():
|
|
||||||
if val is None: r[fld] = False
|
|
||||||
alldata[r['id']] = r
|
|
||||||
data_ids.append(r['id'])
|
|
||||||
del r['id']
|
|
||||||
|
|
||||||
if groupby:
|
none_to_false = lambda record: {k: (False if v is None else v) for k,v in record.iteritems() }
|
||||||
data = self.read(cr, uid, data_ids, [groupby], context=context)
|
fetched_data = map(none_to_false, cr.dictfetchall())
|
||||||
# restore order of the search as read() uses the default _order (this is only for groups, so the footprint of data should be small):
|
|
||||||
data_dict = dict((d['id'], d[groupby] ) for d in data)
|
|
||||||
result = [{'id': i, groupby: data_dict[i]} for i in data_ids]
|
|
||||||
else:
|
|
||||||
result = [{'id': i} for i in data_ids]
|
|
||||||
|
|
||||||
for d in result:
|
data_ids = [r['id'] for r in fetched_data]
|
||||||
if groupby:
|
data_dict = {d['id']: d[first_groupby] for d in self.read(cr, uid, data_ids, [first_groupby], context=context)}
|
||||||
d['__domain'] = [(groupby, '=', alldata[d['id']][groupby] or False)] + domain
|
sorted_data = [{first_groupby: data_dict[id]} for id in data_ids]
|
||||||
if not isinstance(groupby_list, (str, unicode)):
|
|
||||||
if groupby or not context.get('group_by_no_leaf', False):
|
def format_result (fromquery, fromread):
|
||||||
d['__context'] = {'group_by': groupby_list[1:]}
|
result = {
|
||||||
if groupby and groupby in fget:
|
'__domain': [(first_groupby, '=', fromquery[first_groupby] or False)] + domain,
|
||||||
groupby_type = fget[groupby]['type']
|
'__context': {'group_by': groupby[1:]}
|
||||||
if d[groupby] and groupby_type in ('date', 'datetime'):
|
}
|
||||||
groupby_datetime = alldata[d['id']][groupby]
|
result.update(fromquery)
|
||||||
|
result.update(fromread)
|
||||||
|
|
||||||
|
if groupby_type in ('date', 'datetime') and fromquery[first_groupby]:
|
||||||
|
groupby_datetime = fromquery[first_groupby]
|
||||||
if isinstance(groupby_datetime, basestring):
|
if isinstance(groupby_datetime, basestring):
|
||||||
_default = datetime.datetime(1970, 1, 1) # force starts of month
|
groupby_datetime = datetime.datetime.strptime(groupby_datetime, dt_format)
|
||||||
groupby_datetime = dateutil.parser.parse(groupby_datetime, default=_default)
|
|
||||||
tz_convert = groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones
|
|
||||||
if tz_convert:
|
if tz_convert:
|
||||||
groupby_datetime = pytz.timezone(context['tz']).localize(groupby_datetime)
|
groupby_datetime = pytz.timezone(context['tz']).localize(groupby_datetime)
|
||||||
d[groupby] = babel.dates.format_date(
|
result[first_groupby] = babel.dates.format_date(
|
||||||
groupby_datetime, format=display_format, locale=context.get('lang', 'en_US'))
|
groupby_datetime, format=time_display_format, locale=context.get('lang', 'en_US'))
|
||||||
domain_dt_begin = groupby_datetime
|
domain_dt_begin = groupby_datetime
|
||||||
if interval == 'quarter':
|
domain_dt_end = groupby_datetime + time_interval
|
||||||
domain_dt_end = groupby_datetime + dateutil.relativedelta.relativedelta(months=3)
|
|
||||||
elif interval == 'month':
|
|
||||||
domain_dt_end = groupby_datetime + dateutil.relativedelta.relativedelta(months=1)
|
|
||||||
elif interval == 'week':
|
|
||||||
domain_dt_end = groupby_datetime + datetime.timedelta(days=7)
|
|
||||||
elif interval == 'day':
|
|
||||||
domain_dt_end = groupby_datetime + datetime.timedelta(days=1)
|
|
||||||
else:
|
|
||||||
domain_dt_end = groupby_datetime + dateutil.relativedelta.relativedelta(years=1)
|
|
||||||
if tz_convert:
|
if tz_convert:
|
||||||
# the time boundaries were all computed in the apparent TZ of the user,
|
# the time boundaries were all computed in the apparent TZ of the user,
|
||||||
# so we need to convert them to UTC to have proper server-side values.
|
# so we need to convert them to UTC to have proper server-side values.
|
||||||
domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
|
domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
|
||||||
domain_dt_end = domain_dt_end.astimezone(pytz.utc)
|
domain_dt_end = domain_dt_end.astimezone(pytz.utc)
|
||||||
dt_format = DEFAULT_SERVER_DATETIME_FORMAT if groupby_type == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
|
result['__domain'] = [(first_groupby, '>=', domain_dt_begin.strftime(dt_format)),
|
||||||
d['__domain'] = [(groupby, '>=', domain_dt_begin.strftime(dt_format)),
|
(first_groupby, '<', domain_dt_end.strftime(dt_format))] + domain
|
||||||
(groupby, '<', domain_dt_end.strftime(dt_format))] + domain
|
del result['id']
|
||||||
del alldata[d['id']][groupby]
|
return result
|
||||||
d.update(alldata[d['id']])
|
|
||||||
del d['id']
|
|
||||||
|
|
||||||
if groupby and groupby in self._group_by_full:
|
result = map(format_result, fetched_data, sorted_data)
|
||||||
result = self._read_group_fill_results(cr, uid, domain, groupby, groupby_list,
|
|
||||||
|
if first_groupby in self._group_by_full:
|
||||||
|
result = self._read_group_fill_results(cr, uid, domain, first_groupby, groupby,
|
||||||
aggregated_fields, result, read_group_order=order,
|
aggregated_fields, result, read_group_order=order,
|
||||||
context=context)
|
context=context)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _inherits_join_add(self, current_model, parent_model_name, query):
|
def _inherits_join_add(self, current_model, parent_model_name, query):
|
||||||
"""
|
"""
|
||||||
Add missing table SELECT and JOIN clause to ``query`` for reaching the parent table (no duplicates)
|
Add missing table SELECT and JOIN clause to ``query`` for reaching the parent table (no duplicates)
|
||||||
|
|
Loading…
Reference in New Issue