odoo
/
odoo
2
0
Fork 0
Code Pull Requests Releases Activity

[MERGE] this branch rewrite the data loading code from graph view to use the new 'eager' feature of read_group. 

It should improve the graph view performance in a big way (very low number of read_group requests, and that number does not depend on the amount of data)

bzr revid: ged@openerp.com-20140408134917-y0gdmrxa4uzhz9bb
This commit is contained in:
Gery Debongnie 2014-04-08 15:49:17 +02:00
parent cc60736487 53cca3a046
commit 05ec7d45ba
2 changed files with 219 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
addons/web/static/src/js/data.js
View File

@ -27,6 +27,7 @@ instance.web.Query = instance.web.Class.extend({
this._fields = fields;
this._filter = [];
this._context = {};
this._lazy = true;
this._limit = false;
this._offset = 0;
this._order_by = [];
@ -36,6 +37,7 @@ instance.web.Query = instance.web.Class.extend({
var q = new instance.web.Query(this._model, this._fields);
q._context = this._context;
q._filter = this._filter;
q._lazy = this._lazy;
q._limit = this._limit;
q._offset = this._offset;
q._order_by = this._order_by;
@ -51,6 +53,7 @@ instance.web.Query = instance.web.Class.extend({
q._context = new instance.web.CompoundContext(
q._context, to_set.context);
break;
case 'lazy':
case 'limit':
case 'offset':
case 'order_by':
@ -140,6 +143,7 @@ instance.web.Query = instance.web.Class.extend({
domain: this._model.domain(this._filter),
context: ctx,
offset: this._offset,
lazy: this._lazy,
limit: this._limit,
orderby: instance.web.serialize_sort(this._order_by) || false
}).then(function (results) {
@ -148,8 +152,9 @@ instance.web.Query = instance.web.Class.extend({
result.__context = result.__context || {};
result.__context.group_by = result.__context.group_by || [];
_.defaults(result.__context, ctx);
var grouping_fields = self._lazy ? [grouping[0]] : grouping;
return new instance.web.QueryGroup(
self._model.name, grouping[0], result);
self._model.name, grouping_fields, result);
});
});
},
@ -175,6 +180,18 @@ instance.web.Query = instance.web.Class.extend({
if (!domain) { return this; }
return this.clone({filter: domain});
},
/**
* Creates a new query with the provided parameter lazy replacing the current
* query's own.
*
* @param {Boolean} lazy indicates if the read_group should return only the
* first level of groupby records, or should return the records grouped by
* all levels at once (so, it makes only 1 db request).
* @returns {openerp.web.Query}
*/
lazy: function (lazy) {
return this.clone({lazy: lazy});
},
/**
* Creates a new query with the provided limit replacing the current
* query's own limit
@ -213,7 +230,7 @@ instance.web.Query = instance.web.Class.extend({
});
instance.web.QueryGroup = instance.web.Class.extend({
init: function (model, grouping_field, read_group_group) {
init: function (model, grouping_fields, read_group_group) {
// In cases where group_by_no_leaf and no group_by, the result of
// read_group has aggregate fields but no __context or __domain.
// Create default (empty) values for those so that things don't break
@ -221,12 +238,12 @@ instance.web.QueryGroup = instance.web.Class.extend({
{__context: {group_by: []}, __domain: []},
read_group_group);
var raw_field = grouping_field && grouping_field.split(':')[0];
var count_key = (grouping_fields[0] && grouping_fields[0].split(':')[0]) + '_count';
var aggregates = {};
_(fixed_group).each(function (value, key) {
if (key.indexOf('__') === 0
|| key === raw_field
|| key === raw_field + '_count') {
|| _.contains(grouping_fields, key)
|| (key === count_key)) {
return;
}
aggregates[key] = value || 0;
@ -235,15 +252,21 @@ instance.web.QueryGroup = instance.web.Class.extend({
this.model = new instance.web.Model(
model, fixed_group.__context, fixed_group.__domain);
var group_size = fixed_group[raw_field + '_count'] || fixed_group.__count || 0;
var group_size = fixed_group[count_key] || fixed_group.__count || 0;
var leaf_group = fixed_group.__context.group_by.length === 0;
var value = (grouping_fields.length === 1)
? fixed_group[grouping_fields[0]]
: _.map(grouping_fields, function (field) { return fixed_group[field]; });
var grouped_on = (grouping_fields.length === 1)
? grouping_fields[0]
: grouping_fields;
this.attributes = {
folded: !!(fixed_group.__fold),
grouped_on: grouping_field,
grouped_on: grouped_on,
// if terminal group (or no group) and group_by_no_leaf => use group.__count
length: group_size,
value: fixed_group[raw_field],
value: value,
// A group is open-able if it's not a leaf in group_by_no_leaf mode
has_children: !(leaf_group && fixed_group.__context['group_by_no_leaf']),

412
addons/web_graph/static/src/js/pivot_table.js
View File

@ -82,7 +82,9 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
get_values: function (id1, id2, default_values) {
var cell = _.findWhere(this.cells, {x: Math.min(id1, id2), y: Math.max(id1, id2)});
return (cell !== undefined) ? cell.values : (default_values || new Array(this.measures.length));
return (cell !== undefined) ?
cell.values :
(default_values || new Array(this.measures.length));
},
// ----------------------------------------------------------------------
@ -144,12 +146,16 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
get_ancestors: function (header) {
var self = this;
if (!header.children) return [];
return [].concat.apply([], _.map(header.children, function (c) {return self.get_ancestors_and_self(c); }));
return [].concat.apply([], _.map(header.children, function (c) {
return self.get_ancestors_and_self(c);
}));
},
get_ancestors_and_self: function (header) {
var self = this;
return [].concat.apply([header], _.map(header.children, function (c) { return self.get_ancestors_and_self(c); }));
return [].concat.apply([header], _.map(header.children, function (c) {
return self.get_ancestors_and_self(c);
}));
},
get_total: function (header) {
@ -205,54 +211,27 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
expand: function (header_id, groupby) {
var self = this,
header = this.get_header(header_id),
otherRoot = this.get_other_root(header),
fields = otherRoot.groupby.concat(this.measures);
other_root = this.get_other_root(header),
this_gb = [groupby.field],
other_gbs = _.pluck(other_root.groupby, 'field');
if (header.path.length === header.root.groupby.length) {
header.root.groupby.push(groupby);
}
groupby = [groupby].concat(otherRoot.groupby);
return this.get_groups(groupby, fields, header.domain).then(function (groups) {
_.each(groups.reverse(), function (group) {
// make header
var child = self.make_header(group, header);
child.expanded = false;
header.children.splice(0,0, child);
header.root.headers.splice(header.root.headers.indexOf(header) + 1, 0, child);
// make cells
_.each(self.get_ancestors_and_self(group), function (data) {
var values = _.map(self.measures, function (m) {
return data.attributes.aggregates[m.field];
});
var other = _.find(otherRoot.headers, function (h) {
if (header.root === self.cols) {
return _.isEqual(data.path.slice(1), h.path);
} else {
return _.isEqual(_.rest(data.path), h.path);
}
});
if (other) {
self.add_cell(child.id, other.id, values);
}
});
return this.perform_requests(this_gb, other_gbs, header.domain).then(function () {
var data = Array.prototype.slice.call(arguments).slice(other_gbs.length + 1);
_.each(data, function (data_pt) {
self.make_headers_and_cell(
data_pt, header.root.headers, other_root.headers, 1, header.path, true);
});
header.expanded = true;
header.children.forEach(function (child) {
child.expanded = false;
child.root = header.root;
});
});
},
make_header: function (group, parent) {
var title = parent ? group.attributes.value : _t('Total');
return {
id: _.uniqueId(),
path: parent ? parent.path.concat(title) : [],
title: title,
children: [],
domain: parent ? group.model._domain : this.domain,
root: parent ? parent.root : undefined,
};
},
swap_axis: function () {
var temp = this.rows;
this.rows = this.cols;
@ -262,206 +241,191 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
// ----------------------------------------------------------------------
// Data updating methods
// ----------------------------------------------------------------------
// Load the data from the db, using the method this.load_data
// update_data will try to preserve the expand/not expanded status of each
// column/row. If you want to expand all, then set this.cols.headers/this.rows.headers
// to null before calling update_data.
update_data: function () {
var self = this;
update_data: function () {
var self = this;
return this.perform_requests().then (function () {
var data = Array.prototype.slice.call(arguments);
self.no_data = !data[0][0].attributes.length;
if (self.no_data) {
return;
}
var row_headers = [],
col_headers = [];
self.cells = [];
return this.load_data().then (function (result) {
if (result) {
self.no_data = false;
self[self.cols.headers ? 'update_headers' : 'expand_headers'](self.cols, result.col_headers);
self[self.rows.headers ? 'update_headers' : 'expand_headers'](self.rows, result.row_headers);
} else {
self.no_data = true;
}
});
},
var dim_col = self.cols.groupby.length,
i, j, index;
expand_headers: function (root, new_headers) {
root.headers = new_headers;
_.each(root.headers, function (header) {
header.root = root;
header.expanded = (header.children.length > 0);
});
},
update_headers: function (root, new_headers) {
_.each(root.headers, function (header) {
var corresponding_header = _.find(new_headers, function (h) {
return _.isEqual(h.path, header.path);
});
if (corresponding_header && header.expanded) {
corresponding_header.expanded = true;
_.each(corresponding_header.children, function (c) {
c.expanded = false;
});
}
if (corresponding_header && (!header.expanded)) {
corresponding_header.expanded = false;
}
});
var updated_headers = _.filter(new_headers, function (header) {
return (header.expanded !== undefined);
});
_.each(updated_headers, function (header) {
if (!header.expanded) {
header.children = [];
}
header.root = root;
});
root.headers = updated_headers;
},
// ----------------------------------------------------------------------
// Data loading methods
// ----------------------------------------------------------------------
// To obtain all the values required to draw the full table, we have to do
// at least 2 + min(row.groupby.length, col.groupby.length)
// calls to readgroup. To simplify the code, we will always do
// 2 + row.groupby.length calls. For example, if row.groupby = [r1, r2, r3]
// and col.groupby = [c1, c2], then we will make the call with the following
// groupbys: [r1,r2,r3], [c1,r1,r2,r3], [c1,c2,r1,r2,r3], [].
load_data: function () {
var self = this,
cols = this.cols.groupby,
rows = this.rows.groupby,
visible_fields = rows.concat(cols, self.measures);
if (this.measures.length === 0) {
return $.Deferred.resolve().promise();
}
var groupbys = _.map(_.range(cols.length + 1), function (i) {
return cols.slice(0, i).concat(rows);
});
groupbys.push([]);
var get_data_requests = _.map(groupbys, function (groupby) {
return self.get_groups(groupby, visible_fields, self.domain);
});
return $.when.apply(null, get_data_requests).then(function () {
var data = Array.prototype.slice.call(arguments),
row_data = data[0],
col_data = (cols.length !== 0) ? data[data.length - 2] : [],
has_data = data[data.length - 1][0];
return has_data && self.format_data(col_data, row_data, data);
for (i = 0; i < self.rows.groupby.length + 1; i++) {
for (j = 0; j < dim_col + 1; j++) {
index = i*(dim_col + 1) + j;
self.make_headers_and_cell(data[index], row_headers, col_headers, i);
}
}
self.set_headers(row_headers, self.rows);
self.set_headers(col_headers, self.cols);
});
},
get_groups: function (groupbys, fields, domain, path) {
var self = this,
groupby = (groupbys.length) ? groupbys[0] : [];
path = path || [];
return this._query_db(groupby, fields, domain, path).then(function (groups) {
if (groupbys.length > 1) {
var get_subgroups = $.when.apply(null, _.map(groups, function (group) {
return self.get_groups(_.rest(groupbys), fields, group.model._domain, path.concat(group.attributes.value)).then(function (subgroups) {
group.children = subgroups;
});
}));
return get_subgroups.then(function () {
return groups;
});
} else {
return groups;
make_headers_and_cell: function (data_pts, row_headers, col_headers, index, prefix, expand) {
var self = this;
data_pts.forEach(function (data_pt) {
var row_value = (prefix || []).concat(data_pt.attributes.value.slice(0,index));
var col_value = data_pt.attributes.value.slice(index);
if (expand && !_.find(col_headers, function (hdr) {return _.isEqual(col_value, hdr.path);})) {
return;
}
});
var row = self.find_or_create_header(row_headers, row_value, data_pt);
var col = self.find_or_create_header(col_headers, col_value, data_pt);
},
var cell_value = _.map(self.measures, function (m) {
return data_pt.attributes.aggregates[m.field];
});
self.cells.push({
x: Math.min(row.id, col.id),
y: Math.max(row.id, col.id),
values: cell_value
});
});
},
_query_db: function (groupby, fields, domain, path) {
var self = this,
field_ids = _.without(_.pluck(fields, 'field'), '__count'),
fields = _.map(field_ids, function(f) { return self.raw_field(f); });
make_header: function (values) {
return _.extend({
children: [],
domain: this.domain,
expanded: undefined,
id: _.uniqueId(),
path: [],
root: undefined,
title: undefined
}, values || {});
},
return this.model.query(field_ids)
.filter(domain)
.group_by(groupby.field)
.then(function (results) {
var groups = _.filter(results, function (group) {
return group.attributes.length > 0;
});
return _.map(groups, function (g) { return self.format_group(g, path); });
});
},
find_or_create_header: function (headers, path, data_pt) {
var hdr = _.find(headers, function (header) {
return _.isEqual(path, header.path);
});
if (hdr) {
return hdr;
}
if (!path.length) {
hdr = this.make_header({title: _t('Total')});
headers.push(hdr);
return hdr;
}
hdr = this.make_header({
path:path,
domain:data_pt.model._domain,
title: _t(_.last(path))
});
var parent = _.find(headers, function (header) {
return _.isEqual(header.path, _.initial(path, 1));
});
var previous = parent.children.length ? _.last(parent.children) : parent;
headers.splice(headers.indexOf(previous) + 1, 0, hdr);
parent.children.push(hdr);
return hdr;
},
perform_requests: function (group1, group2, domain) {
var self = this,
requests = [],
row_gbs = _.pluck(this.rows.groupby, 'field'),
col_gbs = _.pluck(this.cols.groupby, 'field'),
field_list = row_gbs.concat(col_gbs, _.pluck(this.measures, 'field')),
fields = field_list.map(function (f) { return self.raw_field(f); });
group1 = group1 || row_gbs;
group2 = group2 || col_gbs;
var i,j, groupbys;
for (i = 0; i < group1.length + 1; i++) {
for (j = 0; j < group2.length + 1; j++) {
groupbys = group1.slice(0,i).concat(group2.slice(0,j));
requests.push(self.get_groups(groupbys, fields, domain || self.domain));
}
}
return $.when.apply(null, requests);
},
// set the 'expanded' status of new_headers more or less like root.headers, with root as root
set_headers: function(new_headers, root) {
if (root.headers) {
_.each(root.headers, function (header) {
var corresponding_header = _.find(new_headers, function (h) {
return _.isEqual(h.path, header.path);
});
if (corresponding_header && header.expanded) {
corresponding_header.expanded = true;
_.each(corresponding_header.children, function (c) {
c.expanded = false;
});
}
if (corresponding_header && (!header.expanded)) {
corresponding_header.expanded = false;
corresponding_header.children = [];
}
});
var updated_headers = _.filter(new_headers, function (header) {
return (header.expanded !== undefined);
});
_.each(updated_headers, function (header) {
header.root = root;
});
root.headers = updated_headers;
} else {
root.headers = new_headers;
_.each(root.headers, function (header) {
header.root = root;
header.expanded = (header.children.length > 0);
});
}
return new_headers;
},
get_groups: function (groupbys, fields, domain) {
var self = this;
return this.model.query(_.without(fields, '__count'))
.filter(domain)
.lazy(false)
.group_by(groupbys)
.then(function (groups) {
return groups.filter(function (group) {
return group.attributes.length > 0;
}).map(function (group) {
var attrs = group.attributes,
grouped_on = attrs.grouped_on instanceof Array ? attrs.grouped_on : [attrs.grouped_on],
raw_grouped_on = grouped_on.map(function (f) {
return self.raw_field(f);
});
if (grouped_on.length === 1) {
attrs.value = [attrs.value];
}
attrs.value = _.range(grouped_on.length).map(function (i) {
if (attrs.value[i] === false) {
return _t('Undefined');
} else if (attrs.value[i] instanceof Array) {
return attrs.value[i][1];
}
return attrs.value[i];
});
attrs.aggregates.__count = group.attributes.length;
attrs.grouped_on = raw_grouped_on;
return group;
});
});
},
// if field is a fieldname, returns field, if field is field_id:interval, retuns field_id
raw_field: function (field) {
return field.split(':')[0];
},
// add the path to the group and sanitize the value...
format_group: function (group, current_path) {
var attrs = group.attributes,
value = attrs.value,
grouped_on = attrs.grouped_on ? this.raw_field(attrs.grouped_on) : false;
if (value === false) {
group.attributes.value = _t('Undefined');
} else if (grouped_on && this.fields[grouped_on].type === 'selection') {
var selection = this.fields[grouped_on].selection,
value_lookup = _.where(selection, {0:value});
group.attributes.value = value_lookup ? value_lookup[0][1] : _t('Undefined');
} else if (value instanceof Array) {
group.attributes.value = value[1];
}
group.path = (value !== undefined) ? (current_path || []).concat(group.attributes.value) : [];
group.attributes.aggregates.__count = group.attributes.length;
return group;
},
format_data: function (col_data, row_data, cell_data) {
var self = this,
dim_row = this.rows.groupby.length,
dim_col = this.cols.groupby.length,
col_headers = this.get_ancestors_and_self(this.make_headers(col_data, dim_col)),
row_headers = this.get_ancestors_and_self(this.make_headers(row_data, dim_row));
this.cells = [];
_.each(cell_data, function (data, index) {
self.make_cells(data, index, [], row_headers, col_headers);
}); // not pretty. make it more functional?
return {col_headers: col_headers, row_headers: row_headers};
},
make_headers: function (data, depth, parent) {
var self = this,
main = this.make_header(data, parent);
if (main.path.length < depth) {
main.children = _.map(data.children || data, function (data_pt) {
return self.make_headers (data_pt, depth, main);
});
}
return main;
},
make_cells: function (data, index, current_path, rows, cols) {
var self = this;
_.each(data, function (group) {
var attr = group.attributes,
path = attr.grouped_on ? current_path.concat(attr.value) : current_path,
values = _.map(self.measures, function (measure) { return attr.aggregates[measure.field]; }),
row = _.find(rows, function (header) { return _.isEqual(header.path, path.slice(index)); }),
col = _.find(cols, function (header) { return _.isEqual(header.path, path.slice(0, index)); });
self.add_cell(row.id, col.id, values);
if (group.children) {
self.make_cells (group.children, index, path, rows, cols);
}
});
},
});
})();