odoo
/
odoo
2
0
Fork 0
Code Pull Requests Releases Activity

Merge pull request #5676 from odoo-dev/8.0-memory_optimization-rco 

Reduce memory footprint of server

The tricks used are:
- use a global LRU for `ormcache` (auto-balancing between registries);
- remove unnecessary data structures (binary class hierarchy of models);
- compute some data structures on demand (`_all_columns`);
- optimize field attributes (empty collections may be shared);
- optimize memory size of fields and columns by using slots.

On a database with modules sale, purchase and stock installed, the memory footprint of the registry went from 20.3Mb to 7.4Mb (as measured with heapy). In other words, the memory footprint was reduced to 1/3 !
This commit is contained in:
Raphael Collet 2015-03-23 14:39:21 +01:00
parent e3a80afbaf 5aa23f8280
commit 0ed63d73a6
11 changed files with 569 additions and 379 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
addons/decimal_precision/decimal_precision.py
View File

@ -48,11 +48,6 @@ class decimal_precision(orm.Model):
def clear_cache(self, cr):
"""clear cache and update models. Notify other workers to restart their registry."""
self.precision_get.clear_cache(self)
env = openerp.api.Environment(cr, SUPERUSER_ID, {})
for model in self.pool.values():
for field in model._fields.values():
if field.type == 'float':
field._setup_digits(env)
RegistryManager.signal_registry_change(cr.dbname)
def create(self, cr, uid, data, context=None):

10
openerp/__init__.py
View File

@ -56,11 +56,15 @@ del time
# The hard-coded super-user id (a.k.a. administrator, or root user).
SUPERUSER_ID = 1
def registry(database_name):
def registry(database_name=None):
"""
Return the model registry for the given database. If the registry does not
exist yet, it is created on the fly.
Return the model registry for the given database, or the database mentioned
on the current thread. If the registry does not exist yet, it is created on
the fly.
"""
if database_name is None:
import threading
database_name = threading.currentThread().dbname
return modules.registry.RegistryManager.get(database_name)
#----------------------------------------------------------

4
openerp/addons/base/tests/test_api.py
View File

@ -31,12 +31,10 @@ class TestAPI(common.TransactionCase):
self.assertTrue(ids)
self.assertTrue(partners)
# partners and its contents are instance of the model, and share its ormcache
# partners and its contents are instance of the model
self.assertIsRecordset(partners, 'res.partner')
self.assertIs(partners._ormcache, self.env['res.partner']._ormcache)
for p in partners:
self.assertIsRecord(p, 'res.partner')
self.assertIs(p._ormcache, self.env['res.partner']._ormcache)
self.assertEqual([p.id for p in partners], ids)
self.assertEqual(self.env['res.partner'].browse(ids), partners)

3
openerp/addons/test_new_api/tests/test_related.py
View File

@ -111,9 +111,6 @@ class TestPropertyField(common.TransactionCase):
self.partner._columns.update({
'property_country': fields.property(type='many2one', relation="res.country", string="Country by company"),
})
self.partner._all_columns.update({
'property_country': fields.column_info('property_country', self.partner._columns['property_country'], None, None, None),
})
self.partner._field_create(cr)
partner_id = self.partner.create(cr, alice, {

297
openerp/fields.py
View File

@ -30,12 +30,13 @@ import logging
import pytz
import xmlrpclib
from openerp.tools import float_round, ustr, html_sanitize
from openerp.tools import float_round, frozendict, html_sanitize, ustr
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT as DATE_FORMAT
from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT as DATETIME_FORMAT
DATE_LENGTH = len(date.today().strftime(DATE_FORMAT))
DATETIME_LENGTH = len(datetime.now().strftime(DATETIME_FORMAT))
EMPTY_DICT = frozendict()
_logger = logging.getLogger(__name__)
@ -72,10 +73,25 @@ class MetaField(type):
""" Metaclass for field classes. """
by_type = {}
def __new__(meta, name, bases, attrs):
""" Combine the ``_slots`` dict from parent classes, and determine
`__slots__` for them on the new class.
"""
base_slots = {}
for base in reversed(bases):
base_slots.update(getattr(base, '_slots', ()))
slots = dict(base_slots)
slots.update(attrs.get('_slots', ()))
attrs['__slots__'] = set(slots) - set(base_slots)
attrs['_slots'] = slots
return type.__new__(meta, name, bases, attrs)
def __init__(cls, name, bases, attrs):
super(MetaField, cls).__init__(name, bases, attrs)
if cls.type:
cls.by_type[cls.type] = cls
if cls.type and cls.type not in MetaField.by_type:
MetaField.by_type[cls.type] = cls
# compute class attributes to avoid calling dir() on fields
cls.column_attrs = []
@ -253,56 +269,81 @@ class Field(object):
"""
__metaclass__ = MetaField
_attrs = None # dictionary with all field attributes
_free_attrs = None # list of semantic-free attribute names
type = None # type of the field (string)
relational = False # whether the field is a relational one
automatic = False # whether the field is automatically created ("magic" field)
inherited = False # whether the field is inherited (_inherits)
column = None # the column corresponding to the field
setup_done = False # whether the field has been set up
_slots = {
'_attrs': EMPTY_DICT, # dictionary of field attributes; it contains:
# - all attributes after __init__()
# - free attributes only after set_class_name()
name = None # name of the field
type = None # type of the field (string)
relational = False # whether the field is a relational one
model_name = None # name of the model of this field
comodel_name = None # name of the model of values (if relational)
inverse_fields = None # list of inverse fields (objects)
'automatic': False, # whether the field is automatically created ("magic" field)
'inherited': False, # whether the field is inherited (_inherits)
'column': None, # the column corresponding to the field
'setup_done': False, # whether the field has been set up
store = True # whether the field is stored in database
index = False # whether the field is indexed in database
manual = False # whether the field is a custom field
copy = True # whether the field is copied over by BaseModel.copy()
depends = () # collection of field dependencies
recursive = False # whether self depends on itself
compute = None # compute(recs) computes field on recs
compute_sudo = False # whether field should be recomputed as admin
inverse = None # inverse(recs) inverses field on recs
search = None # search(recs, operator, value) searches on self
related = None # sequence of field names, for related fields
related_sudo = True # whether related fields should be read as admin
company_dependent = False # whether `self` is company-dependent (property field)
default = None # default(recs) returns the default value
'name': None, # name of the field
'model_name': None, # name of the model of this field
'comodel_name': None, # name of the model of values (if relational)
string = None # field label
help = None # field tooltip
readonly = False
required = False
states = None
groups = False # csv list of group xml ids
change_default = None # whether the field may trigger a "user-onchange"
deprecated = None # whether the field is ... deprecated
'store': True, # whether the field is stored in database
'index': False, # whether the field is indexed in database
'manual': False, # whether the field is a custom field
'copy': True, # whether the field is copied over by BaseModel.copy()
'depends': (), # collection of field dependencies
'recursive': False, # whether self depends on itself
'compute': None, # compute(recs) computes field on recs
'compute_sudo': False, # whether field should be recomputed as admin
'inverse': None, # inverse(recs) inverses field on recs
'search': None, # search(recs, operator, value) searches on self
'related': None, # sequence of field names, for related fields
'related_sudo': True, # whether related fields should be read as admin
'company_dependent': False, # whether `self` is company-dependent (property field)
'default': None, # default(recs) returns the default value
'string': None, # field label
'help': None, # field tooltip
'readonly': False, # whether the field is readonly
'required': False, # whether the field is required
'states': None, # set readonly and required depending on state
'groups': None, # csv list of group xml ids
'change_default': False, # whether the field may trigger a "user-onchange"
'deprecated': None, # whether the field is deprecated
'inverse_fields': (), # collection of inverse fields (objects)
'computed_fields': (), # fields computed with the same method as self
'related_field': None, # corresponding related field
'_triggers': (), # invalidation and recomputation triggers
}
def __init__(self, string=None, **kwargs):
kwargs['string'] = string
self._attrs = {key: val for key, val in kwargs.iteritems() if val is not None}
self._free_attrs = []
attrs = {key: val for key, val in kwargs.iteritems() if val is not None}
self._attrs = attrs or EMPTY_DICT
# self._triggers is a set of pairs (field, path) that represents the
# computed fields that depend on `self`. When `self` is modified, it
# invalidates the cache of each `field`, and registers the records to
# recompute based on `path`. See method `modified` below for details.
self._triggers = set()
self.inverse_fields = []
def __getattr__(self, name):
""" Access non-slot field attribute. """
try:
return self._attrs[name]
except KeyError:
raise AttributeError(name)
def __setattr__(self, name, value):
""" Set slot or non-slot field attribute. """
try:
object.__setattr__(self, name, value)
except AttributeError:
if self._attrs:
self._attrs[name] = value
else:
self._attrs = {name: value} # replace EMPTY_DICT
def __delattr__(self, name):
""" Remove non-slot field attribute. """
try:
del self._attrs[name]
except KeyError:
raise AttributeError(name)
def new(self, **kwargs):
""" Return a field of the same type as `self`, with its own parameters. """
@ -310,6 +351,10 @@ class Field(object):
def set_class_name(self, cls, name):
""" Assign the model class and field name of `self`. """
self_attrs = self._attrs
for attr, value in self._slots.iteritems():
setattr(self, attr, value)
self.model_name = cls._name
self.name = name
@ -320,7 +365,7 @@ class Field(object):
attrs.update(field._attrs)
else:
attrs.clear()
attrs.update(self._attrs) # necessary in case self is not in cls
attrs.update(self_attrs) # necessary in case self is not in cls
# initialize `self` with `attrs`
if attrs.get('compute'):
@ -338,8 +383,6 @@ class Field(object):
attrs.pop('store', None)
for attr, value in attrs.iteritems():
if not hasattr(self, attr):
self._free_attrs.append(attr)
setattr(self, attr, value)
if not self.string and not self.related:
@ -478,10 +521,9 @@ class Field(object):
if not getattr(self, attr):
setattr(self, attr, getattr(field, prop))
for attr in field._free_attrs:
if attr not in self._free_attrs:
self._free_attrs.append(attr)
setattr(self, attr, getattr(field, attr))
for attr, value in field._attrs.iteritems():
if attr not in self._attrs:
setattr(self, attr, value)
# special case for states: copy it only for inherited fields
if not self.states and self.inherited:
@ -533,6 +575,16 @@ class Field(object):
#
# Setup of field triggers
#
# The triggers is a collection of pairs (field, path) of computed fields
# that depend on `self`. When `self` is modified, it invalidates the cache
# of each `field`, and registers the records to recompute based on `path`.
# See method `modified` below for details.
#
def add_trigger(self, trigger):
""" Add a recomputation trigger on `self`. """
if trigger not in self._triggers:
self._triggers += (trigger,)
def setup_triggers(self, env):
""" Add the necessary triggers to invalidate/recompute `self`. """
@ -561,12 +613,12 @@ class Field(object):
continue
#_logger.debug("Add trigger on %s to recompute %s", field, self)
field._triggers.add((self, '.'.join(path0 or ['id'])))
field.add_trigger((self, '.'.join(path0 or ['id'])))
# add trigger on inverse fields, too
for invf in field.inverse_fields:
#_logger.debug("Add trigger on %s to recompute %s", invf, self)
invf._triggers.add((self, '.'.join(path0 + [head])))
invf.add_trigger((self, '.'.join(path0 + [head])))
# recursively traverse the dependency
if tail:
@ -648,8 +700,8 @@ class Field(object):
args = {}
for attr, prop in self.column_attrs:
args[attr] = getattr(self, prop)
for attr in self._free_attrs:
args[attr] = getattr(self, attr)
for attr, value in self._attrs.iteritems():
args[attr] = value
if self.company_dependent:
# company-dependent fields are mapped to former property fields
@ -959,10 +1011,11 @@ class Boolean(Field):
class Integer(Field):
type = 'integer'
group_operator = None # operator for aggregating values
_slots = {
'group_operator': None, # operator for aggregating values
}
_related_group_operator = property(attrgetter('group_operator'))
_column_group_operator = property(attrgetter('group_operator'))
def convert_to_cache(self, value, record, validate=True):
@ -990,27 +1043,31 @@ class Float(Field):
cursor and returning a pair (total, decimal)
"""
type = 'float'
_digits = None # digits argument passed to class initializer
digits = None # digits as computed by setup()
group_operator = None # operator for aggregating values
_slots = {
'_digits': None, # digits argument passed to class initializer
'group_operator': None, # operator for aggregating values
}
def __init__(self, string=None, digits=None, **kwargs):
super(Float, self).__init__(string=string, _digits=digits, **kwargs)
@property
def digits(self):
if callable(self._digits):
with registry().cursor() as cr:
return self._digits(cr)
else:
return self._digits
def _setup_digits(self, env):
""" Setup the digits for `self` and its corresponding column """
self.digits = self._digits(env.cr) if callable(self._digits) else self._digits
if self.digits:
assert isinstance(self.digits, (tuple, list)) and len(self.digits) >= 2, \
"Float field %s with digits %r, expecting (total, decimal)" % (self, self.digits)
if self.column:
self.column.digits_change(env.cr)
pass
def _setup_regular(self, env):
super(Float, self)._setup_regular(env)
self._setup_digits(env)
_related_digits = property(attrgetter('digits'))
_related__digits = property(attrgetter('_digits'))
_related_group_operator = property(attrgetter('group_operator'))
_description_digits = property(attrgetter('digits'))
@ -1021,15 +1078,16 @@ class Float(Field):
def convert_to_cache(self, value, record, validate=True):
# apply rounding here, otherwise value in cache may be wrong!
if self.digits:
return float_round(float(value or 0.0), precision_digits=self.digits[1])
else:
return float(value or 0.0)
value = float(value or 0.0)
digits = self.digits
return float_round(value, precision_digits=digits[1]) if digits else value
class _String(Field):
""" Abstract class for string fields. """
translate = False
_slots = {
'translate': False, # whether the field is translated
}
_column_translate = property(attrgetter('translate'))
_related_translate = property(attrgetter('translate'))
@ -1044,17 +1102,19 @@ class Char(_String):
:param bool translate: whether the values of this field can be translated
"""
type = 'char'
size = None
_slots = {
'size': None, # maximum size of values (deprecated)
}
_column_size = property(attrgetter('size'))
_related_size = property(attrgetter('size'))
_description_size = property(attrgetter('size'))
def _setup_regular(self, env):
super(Char, self)._setup_regular(env)
assert isinstance(self.size, (NoneType, int)), \
"Char field %s with non-integer size %r" % (self, self.size)
_column_size = property(attrgetter('size'))
_related_size = property(attrgetter('size'))
_description_size = property(attrgetter('size'))
def convert_to_cache(self, value, record, validate=True):
if value is None or value is False:
return False
@ -1075,8 +1135,10 @@ class Text(_String):
class Html(_String):
type = 'html'
sanitize = True # whether value must be sanitized
strip_style = False # whether to strip style attributes
_slots = {
'sanitize': True, # whether value must be sanitized
'strip_style': False, # whether to strip style attributes
}
_column_sanitize = property(attrgetter('sanitize'))
_related_sanitize = property(attrgetter('sanitize'))
@ -1245,8 +1307,9 @@ class Selection(Field):
<field-incremental-definition>`.
"""
type = 'selection'
selection = None # [(value, string), ...], function or method name
selection_add = None # [(value, string), ...]
_slots = {
'selection': None, # [(value, string), ...], function or method name
}
def __init__(self, selection=None, string=None, **kwargs):
if callable(selection):
@ -1337,20 +1400,18 @@ class Selection(Field):
class Reference(Selection):
type = 'reference'
size = None
_slots = {
'size': None, # maximum size of values (deprecated)
}
def __init__(self, selection=None, string=None, **kwargs):
super(Reference, self).__init__(selection=selection, string=string, **kwargs)
_related_size = property(attrgetter('size'))
_column_size = property(attrgetter('size'))
def _setup_regular(self, env):
super(Reference, self)._setup_regular(env)
assert isinstance(self.size, (NoneType, int)), \
"Reference field %s with non-integer size %r" % (self, self.size)
_related_size = property(attrgetter('size'))
_column_size = property(attrgetter('size'))
def convert_to_cache(self, value, record, validate=True):
if isinstance(value, BaseModel):
if ((not validate or value._name in self.get_values(record.env))
@ -1376,8 +1437,10 @@ class Reference(Selection):
class _Relational(Field):
""" Abstract class for relational fields. """
relational = True
domain = None # domain for searching values
context = None # context for searching values
_slots = {
'domain': [], # domain for searching values
'context': {}, # context for searching values
}
def _setup_regular(self, env):
super(_Relational, self)._setup_regular(env)
@ -1445,9 +1508,11 @@ class Many2one(_Relational):
fields or field extensions.
"""
type = 'many2one'
ondelete = 'set null' # what to do when value is deleted
auto_join = False # whether joins are generated upon search
delegate = False # whether self implements delegation
_slots = {
'ondelete': 'set null', # what to do when value is deleted
'auto_join': False, # whether joins are generated upon search
'delegate': False, # whether self implements delegation
}
def __init__(self, comodel_name=None, string=None, **kwargs):
super(Many2one, self).__init__(comodel_name=comodel_name, string=string, **kwargs)
@ -1650,10 +1715,12 @@ class One2many(_RelationalMulti):
the case of related fields or field extensions.
"""
type = 'one2many'
inverse_name = None # name of the inverse field
auto_join = False # whether joins are generated upon search
limit = None # optional limit to use upon read
copy = False # o2m are not copied by default
_slots = {
'inverse_name': None, # name of the inverse field
'auto_join': False, # whether joins are generated upon search
'limit': None, # optional limit to use upon read
'copy': False, # o2m are not copied by default
}
def __init__(self, comodel_name=None, inverse_name=None, string=None, **kwargs):
super(One2many, self).__init__(
@ -1674,8 +1741,8 @@ class One2many(_RelationalMulti):
# (res_model/res_id pattern). Only inverse the field if this is
# a `Many2one` field.
if isinstance(invf, Many2one):
self.inverse_fields.append(invf)
invf.inverse_fields.append(self)
self.inverse_fields += (invf,)
invf.inverse_fields += (self,)
_description_relation_field = property(attrgetter('inverse_name'))
@ -1715,10 +1782,12 @@ class Many2many(_RelationalMulti):
"""
type = 'many2many'
relation = None # name of table
column1 = None # column of table referring to model
column2 = None # column of table referring to comodel
limit = None # optional limit to use upon read
_slots = {
'relation': None, # name of table
'column1': None, # column of table referring to model
'column2': None, # column of table referring to comodel
'limit': None, # optional limit to use upon read
}
def __init__(self, comodel_name=None, relation=None, column1=None, column2=None,
string=None, **kwargs):
@ -1746,8 +1815,8 @@ class Many2many(_RelationalMulti):
# if inverse field has already been setup, it is present in m2m
invf = m2m.get((self.relation, self.column2, self.column1))
if invf:
self.inverse_fields.append(invf)
invf.inverse_fields.append(self)
self.inverse_fields += (invf,)
invf.inverse_fields += (self,)
else:
# add self in m2m, so that its inverse field can find it
m2m[(self.relation, self.column1, self.column2)] = self
@ -1768,15 +1837,15 @@ class Serialized(Field):
class Id(Field):
""" Special case for field 'id'. """
store = True
#: Can't write this!
readonly = True
def __init__(self, string=None, **kwargs):
super(Id, self).__init__(type='integer', string=string, **kwargs)
type = 'integer'
_slots = {
'string': 'ID',
'store': True,
'readonly': True,
}
def to_column(self):
self.column = fields.integer('ID')
self.column = fields.integer(self.string)
return self.column
def __get__(self, record, owner):
@ -1790,7 +1859,7 @@ class Id(Field):
raise TypeError("field 'id' cannot be assigned")
# imported here to avoid dependency cycle issues
from openerp import SUPERUSER_ID
from openerp import SUPERUSER_ID, registry
from .exceptions import Warning, AccessError, MissingError
from .models import BaseModel, MAGIC_COLUMNS
from .osv import fields

143
openerp/models.py
View File

@ -336,13 +336,6 @@ class BaseModel(object):
# field_column_obj, origina_parent_model), ... }
_inherit_fields = {}
# Mapping field name/column_info object
# This is similar to _inherit_fields but:
# 1. includes self fields,
# 2. uses column_info instead of a triple.
# Warning: _all_columns is deprecated, use _fields instead
_all_columns = {}
_table = None
_log_create = False
_sql_constraints = []
@ -491,7 +484,6 @@ class BaseModel(object):
"""
field = cls._fields.pop(name)
cls._columns.pop(name, None)
cls._all_columns.pop(name, None)
if hasattr(cls, name):
delattr(cls, name)
return field
@ -570,11 +562,22 @@ class BaseModel(object):
"""
# IMPORTANT: the registry contains an instance for each model. The class
# of each model carries inferred metadata that is shared among the
# model's instances for this registry, but not among registries. Hence
# we cannot use that "registry class" for combining model classes by
# inheritance, since it confuses the metadata inference process.
# The model's class inherits from cls and the classes of the inherited
# models. All those classes are combined in a flat hierarchy:
#
# Model the base class of all models
# / | \
# cls c2 c1 the classes defined in modules
# \ | /
# ModelClass the final class of the model
# / | \
# model recordset ... the class' instances
#
# The registry contains the instance `model`. Its class, `ModelClass`,
# carries inferred metadata that is shared between all the model's
# instances for this registry only. When we '_inherit' from another
# model, we do not inherit its `ModelClass`, but this class' parents.
# This is a limitation of the inheritance mechanism.
# Keep links to non-inherited constraints in cls; this is useful for
# instance when exporting translations
@ -591,65 +594,54 @@ class BaseModel(object):
# determine the module that introduced the model
original_module = pool[name]._original_module if name in parents else cls._module
# build the class hierarchy for the model
# determine all the classes the model should inherit from
bases = [cls]
hierarchy = cls
for parent in parents:
if parent not in pool:
raise TypeError('The model "%s" specifies an unexisting parent class "%s"\n'
'You may need to add a dependency on the parent class\' module.' % (name, parent))
parent_model = pool[parent]
parent_class = type(pool[parent])
bases += parent_class.__bases__
hierarchy = type(name, (hierarchy, parent_class), {'_register': False})
# do no use the class of parent_model, since that class contains
# inferred metadata; use its ancestor instead
parent_class = type(parent_model).__base__
# order bases following the mro of class hierarchy
bases = [base for base in hierarchy.mro() if base in bases]
inherits = dict(parent_class._inherits)
inherits.update(cls._inherits)
# determine the attributes of the model's class
inherits = {}
depends = {}
constraints = {}
sql_constraints = []
depends = dict(parent_class._depends)
for m, fs in cls._depends.iteritems():
depends[m] = depends.get(m, []) + fs
for base in reversed(bases):
inherits.update(base._inherits)
old_constraints = parent_class._constraints
new_constraints = cls._constraints
# filter out from old_constraints the ones overridden by a
# constraint with the same function name in new_constraints
constraints = new_constraints + [oldc
for oldc in old_constraints
if not any(newc[2] == oldc[2] and same_name(newc[0], oldc[0])
for newc in new_constraints)
]
for mname, fnames in base._depends.iteritems():
depends[mname] = depends.get(mname, []) + fnames
sql_constraints = cls._sql_constraints + \
parent_class._sql_constraints
for cons in base._constraints:
# cons may override a constraint with the same function name
constraints[getattr(cons[0], '__name__', id(cons[0]))] = cons
attrs = {
'_name': name,
'_register': False,
'_inherits': inherits,
'_depends': depends,
'_constraints': constraints,
'_sql_constraints': sql_constraints,
}
cls = type(name, (cls, parent_class), attrs)
sql_constraints += base._sql_constraints
# introduce the "registry class" of the model;
# duplicate some attributes so that the ORM can modify them
attrs = {
# build the actual class of the model
ModelClass = type(name, tuple(bases), {
'_name': name,
'_register': False,
'_columns': None, # recomputed in _setup_fields()
'_defaults': None, # recomputed in _setup_base()
'_fields': frozendict(), # idem
'_inherits': dict(cls._inherits),
'_depends': dict(cls._depends),
'_constraints': list(cls._constraints),
'_sql_constraints': list(cls._sql_constraints),
'_inherits': inherits,
'_depends': depends,
'_constraints': constraints.values(),
'_sql_constraints': sql_constraints,
'_original_module': original_module,
}
cls = type(cls._name, (cls,), attrs)
})
# instantiate the model, and initialize it
model = object.__new__(cls)
model = object.__new__(ModelClass)
model.__init__(pool, cr)
return model
@ -660,8 +652,6 @@ class BaseModel(object):
# process store of low-level function fields
for fname, column in cls._columns.iteritems():
if hasattr(column, 'digits_change'):
column.digits_change(cr)
# filter out existing store about this field
pool._store_function[cls._name] = [
stored
@ -827,9 +817,6 @@ class BaseModel(object):
"TransientModels must have log_access turned on, " \
"in order to implement their access rights policy"
# prepare ormcache, which must be shared by all instances of the model
cls._ormcache = {}
@api.model
@ormcache()
def _is_an_ordinary_table(self):
@ -1827,7 +1814,7 @@ class BaseModel(object):
``tools.ormcache`` or ``tools.ormcache_multi``.
"""
try:
self._ormcache.clear()
self.pool.cache.clear_prefix((self.pool.db_name, self._name))
self.pool._any_cache_cleared = True
except AttributeError:
pass
@ -2901,7 +2888,7 @@ class BaseModel(object):
@classmethod
def _inherits_reload(cls):
""" Recompute the _inherit_fields and _all_columns mappings. """
""" Recompute the _inherit_fields mapping. """
cls._inherit_fields = struct = {}
for parent_model, parent_field in cls._inherits.iteritems():
parent = cls.pool[parent_model]
@ -2911,19 +2898,17 @@ class BaseModel(object):
for name, source in parent._inherit_fields.iteritems():
struct[name] = (parent_model, parent_field, source[2], source[3])
# old-api stuff
cls._all_columns = cls._get_column_infos()
@classmethod
def _get_column_infos(cls):
"""Returns a dict mapping all fields names (direct fields and
inherited field via _inherits) to a ``column_info`` struct
giving detailed columns """
@property
def _all_columns(self):
""" Returns a dict mapping all fields names (self fields and inherited
field via _inherits) to a ``column_info`` object giving detailed column
information. This property is deprecated, use ``_fields`` instead.
"""
result = {}
# do not inverse for loops, since local fields may hide inherited ones!
for k, (parent, m2o, col, original_parent) in cls._inherit_fields.iteritems():
for k, (parent, m2o, col, original_parent) in self._inherit_fields.iteritems():
result[k] = fields.column_info(k, col, parent, m2o, original_parent)
for k, col in cls._columns.iteritems():
for k, col in self._columns.iteritems():
result[k] = fields.column_info(k, col)
return result
@ -3000,14 +2985,15 @@ class BaseModel(object):
if column:
cls._columns[name] = column
# group fields by compute to determine field.computed_fields
fields_by_compute = defaultdict(list)
# determine field.computed_fields
computed_fields = defaultdict(list)
for field in cls._fields.itervalues():
if field.compute:
field.computed_fields = fields_by_compute[field.compute]
field.computed_fields.append(field)
else:
field.computed_fields = []
computed_fields[field.compute].append(field)
for fields in computed_fields.itervalues():
for field in fields:
field.computed_fields = fields
@api.model
def _setup_complete(self):
@ -3025,9 +3011,10 @@ class BaseModel(object):
model = self.env[model_name]
for field_name in field_names:
field = model._fields[field_name]
field._triggers.update(triggers)
for trigger in triggers:
field.add_trigger(trigger)
# determine old-api cls._inherit_fields and cls._all_columns
# determine old-api structures about inherited fields
cls._inherits_reload()
# register stuff about low-level function fields

22
openerp/modules/registry.py
View File

@ -104,6 +104,10 @@ class Registry(Mapping):
""" Same as ``self[model_name]``. """
return self.models[model_name]
@lazy_property
def cache(self):
return RegistryManager.cache
@lazy_property
def pure_function_fields(self):
""" Return the list of pure function fields (field objects) """
@ -287,6 +291,7 @@ class RegistryManager(object):
"""
_registries = None
_cache = None
_lock = threading.RLock()
_saved_lock = None
@ -300,13 +305,26 @@ class RegistryManager(object):
# cannot specify the memory limit soft on windows...
size = 42
else:
# On average, a clean registry take 25MB of memory + cache
avgsz = 30 * 1024 * 1024
# A registry takes 10MB of memory on average, so we reserve
# 10Mb (registry) + 5Mb (working memory) per registry
avgsz = 15 * 1024 * 1024
size = int(config['limit_memory_soft'] / avgsz)
cls._registries = LRU(size)
return cls._registries
@classproperty
def cache(cls):
""" Return the global LRU ormcache. Its keys are tuples with the
following structure: (db_name, model_name, method, args...).
"""
with cls.lock():
if cls._cache is None:
# we allocate 8192 cache entries per registry
size = 8192 * cls.registries.count
cls._cache = LRU(size)
return cls._cache
@classmethod
def lock(cls):
""" Return the current registry lock. """

355
openerp/osv/fields.py
View File

@ -48,10 +48,11 @@ from psycopg2 import Binary
import openerp
import openerp.tools as tools
from openerp.tools.translate import _
from openerp.tools import float_round, float_repr
from openerp.tools import html_sanitize
from openerp.tools import float_repr, float_round, frozendict, html_sanitize
import simplejson
from openerp import SUPERUSER_ID
from openerp import SUPERUSER_ID, registry
EMPTY_DICT = frozendict()
_logger = logging.getLogger(__name__)
@ -73,7 +74,6 @@ class _column(object):
_classic_read = True
_classic_write = True
_auto_join = False
_prefetch = True
_properties = False
_type = 'unknown'
_obj = None
@ -84,59 +84,64 @@ class _column(object):
_symbol_get = None
_deprecated = False
copy = True # whether value is copied by BaseModel.copy()
string = None
help = ""
required = False
readonly = False
_domain = []
_context = {}
states = None
priority = 0
change_default = False
size = None
ondelete = None
translate = False
select = False
manual = False
write = False
read = False
selectable = True
group_operator = False
groups = False # CSV list of ext IDs of groups
deprecated = False # Optional deprecation warning
__slots__ = [
'copy', # whether value is copied by BaseModel.copy()
'string',
'help',
'required',
'readonly',
'_domain',
'_context',
'states',
'priority',
'change_default',
'size',
'ondelete',
'translate',
'select',
'manual',
'write',
'read',
'selectable',
'group_operator',
'groups', # CSV list of ext IDs of groups
'deprecated', # Optional deprecation warning
'_args',
'_prefetch',
]
def __init__(self, string='unknown', required=False, readonly=False, domain=None, context=None, states=None, priority=0, change_default=False, size=None, ondelete=None, translate=False, select=False, manual=False, **args):
def __init__(self, string='unknown', required=False, readonly=False, domain=[], context={}, states=None, priority=0, change_default=False, size=None, ondelete=None, translate=False, select=False, manual=False, **args):
"""
The 'manual' keyword argument specifies if the field is a custom one.
It corresponds to the 'state' column in ir_model_fields.
"""
args0 = {
'string': string,
'help': args.pop('help', None),
'required': required,
'readonly': readonly,
'_domain': domain,
'_context': context,
'states': states,
'priority': priority,
'change_default': change_default,
'size': size,
'ondelete': ondelete.lower() if ondelete else None,
'translate': translate,
'select': select,
'manual': manual,
'group_operator': args.pop('group_operator', None),
'groups': args.pop('groups', None),
'deprecated': args.pop('deprecated', None),
}
for key, val in args0.iteritems():
if val:
setattr(self, key, val)
# add parameters and default values
args['copy'] = args.get('copy', True)
args['string'] = string
args['help'] = args.get('help', '')
args['required'] = required
args['readonly'] = readonly
args['_domain'] = domain
args['_context'] = context
args['states'] = states
args['priority'] = priority
args['change_default'] = change_default
args['size'] = size
args['ondelete'] = ondelete.lower() if ondelete else None
args['translate'] = translate
args['select'] = select
args['manual'] = manual
args['write'] = args.get('write', False)
args['read'] = args.get('read', False)
args['selectable'] = args.get('selectable', True)
args['group_operator'] = args.get('group_operator', None)
args['groups'] = args.get('groups', None)
args['deprecated'] = args.get('deprecated', None)
args['_prefetch'] = args.get('_prefetch', True)
self._args = args
self._args = EMPTY_DICT
for key, val in args.iteritems():
setattr(self, key, val)
@ -144,6 +149,30 @@ class _column(object):
if not self._classic_write or self.deprecated or self.manual:
self._prefetch = False
def __getattr__(self, name):
""" Access a non-slot attribute. """
try:
return self._args[name]
except KeyError:
raise AttributeError(name)
def __setattr__(self, name, value):
""" Set a slot or non-slot attribute. """
try:
object.__setattr__(self, name, value)
except AttributeError:
if self._args:
self._args[name] = value
else:
self._args = {name: value} # replace EMPTY_DICT
def __delattr__(self, name):
""" Remove a non-slot attribute. """
try:
del self._args[name]
except KeyError:
raise AttributeError(name)
def new(self, _computed_field=False, **args):
""" Return a column like `self` with the given parameters; the parameter
`_computed_field` tells whether the corresponding field is computed.
@ -223,6 +252,7 @@ class boolean(_column):
_symbol_c = '%s'
_symbol_f = bool
_symbol_set = (_symbol_c, _symbol_f)
__slots__ = []
def __init__(self, string='unknown', required=False, **args):
super(boolean, self).__init__(string=string, required=required, **args)
@ -238,6 +268,7 @@ class integer(_column):
_symbol_f = lambda x: int(x or 0)
_symbol_set = (_symbol_c, _symbol_f)
_symbol_get = lambda self,x: x or 0
__slots__ = []
def __init__(self, string='unknown', required=False, **args):
super(integer, self).__init__(string=string, required=required, **args)
@ -245,6 +276,7 @@ class integer(_column):
class reference(_column):
_type = 'reference'
_classic_read = False # post-process to handle missing target
__slots__ = ['selection']
def __init__(self, string, selection, size=None, **args):
if callable(selection):
@ -297,6 +329,7 @@ def _symbol_set_char(self, symb):
class char(_column):
_type = 'char'
__slots__ = ['_symbol_f', '_symbol_set', '_symbol_set_char']
def __init__(self, string="unknown", size=None, **args):
_column.__init__(self, string=string, size=size or None, **args)
@ -306,11 +339,13 @@ class char(_column):
class text(_column):
_type = 'text'
__slots__ = []
class html(text):
_type = 'html'
_symbol_c = '%s'
__slots__ = ['_sanitize', '_strip_style', '_symbol_f', '_symbol_set']
def _symbol_set_html(self, value):
if value is None or value is False:
@ -334,39 +369,47 @@ class html(text):
import __builtin__
def _symbol_set_float(self, x):
result = __builtin__.float(x or 0.0)
digits = self.digits
if digits:
precision, scale = digits
result = float_repr(float_round(result, precision_digits=scale), precision_digits=scale)
return result
class float(_column):
_type = 'float'
_symbol_c = '%s'
_symbol_f = lambda x: __builtin__.float(x or 0.0)
_symbol_set = (_symbol_c, _symbol_f)
_symbol_get = lambda self,x: x or 0.0
__slots__ = ['_digits', '_digits_compute', '_symbol_f', '_symbol_set']
@property
def digits(self):
if self._digits_compute:
with registry().cursor() as cr:
return self._digits_compute(cr)
else:
return self._digits
def __init__(self, string='unknown', digits=None, digits_compute=None, required=False, **args):
_column.__init__(self, string=string, required=required, **args)
self.digits = digits
# synopsis: digits_compute(cr) -> (precision, scale)
self.digits_compute = digits_compute
def new(self, _computed_field=False, **args):
# float columns are database-dependent, so always recreate them
return type(self)(**args)
self._digits = digits
self._digits_compute = digits_compute
self._symbol_f = lambda x: _symbol_set_float(self, x)
self._symbol_set = (self._symbol_c, self._symbol_f)
def to_field_args(self):
args = super(float, self).to_field_args()
args['digits'] = self.digits_compute or self.digits
args['digits'] = self._digits_compute or self._digits
return args
def digits_change(self, cr):
if self.digits_compute:
self.digits = self.digits_compute(cr)
if self.digits:
precision, scale = self.digits
self._symbol_set = ('%s', lambda x: float_repr(float_round(__builtin__.float(x or 0.0),
precision_digits=scale),
precision_digits=scale))
pass
class date(_column):
_type = 'date'
__slots__ = []
MONTHS = [
('01', 'January'),
@ -457,6 +500,7 @@ class date(_column):
class datetime(_column):
_type = 'datetime'
__slots__ = []
MONTHS = [
('01', 'January'),
@ -526,7 +570,7 @@ class datetime(_column):
class binary(_column):
_type = 'binary'
_symbol_c = '%s'
_classic_read = False
# Binary values may be byte strings (python 2.6 byte array), but
# the legacy OpenERP convention is to transfer and store binaries
@ -534,17 +578,16 @@ class binary(_column):
# unicode in some circumstances, hence the str() cast in symbol_f.
# This str coercion will only work for pure ASCII unicode strings,
# on purpose - non base64 data must be passed as a 8bit byte strings.
_symbol_c = '%s'
_symbol_f = lambda symb: symb and Binary(str(symb)) or None
_symbol_set = (_symbol_c, _symbol_f)
_symbol_get = lambda self, x: x and str(x)
_classic_read = False
_prefetch = False
__slots__ = ['filters']
def __init__(self, string='unknown', filters=None, **args):
_column.__init__(self, string=string, **args)
self.filters = filters
args['_prefetch'] = args.get('_prefetch', False)
_column.__init__(self, string=string, filters=filters, **args)
def get(self, cr, obj, ids, name, user=None, context=None, values=None):
if not context:
@ -572,13 +615,13 @@ class binary(_column):
class selection(_column):
_type = 'selection'
__slots__ = ['selection']
def __init__(self, selection, string='unknown', **args):
if callable(selection):
from openerp import api
selection = api.expected(api.cr_uid_context, selection)
_column.__init__(self, string=string, **args)
self.selection = selection
_column.__init__(self, string=string, selection=selection, **args)
def to_field_args(self):
args = super(selection, self).to_field_args()
@ -639,9 +682,10 @@ class many2one(_column):
_symbol_f = lambda x: x or None
_symbol_set = (_symbol_c, _symbol_f)
ondelete = 'set null'
__slots__ = ['_obj', '_auto_join']
def __init__(self, obj, string='unknown', auto_join=False, **args):
args['ondelete'] = args.get('ondelete', 'set null')
_column.__init__(self, string=string, **args)
self._obj = obj
self._auto_join = auto_join
@ -687,13 +731,14 @@ class many2one(_column):
class one2many(_column):
_classic_read = False
_classic_write = False
_prefetch = False
_type = 'one2many'
# one2many columns are not copied by default
copy = False
__slots__ = ['_obj', '_fields_id', '_limit', '_auto_join']
def __init__(self, obj, fields_id, string='unknown', limit=None, auto_join=False, **args):
# one2many columns are not copied by default
args['copy'] = args.get('copy', False)
args['_prefetch'] = args.get('_prefetch', False)
_column.__init__(self, string=string, **args)
self._obj = obj
self._fields_id = fields_id
@ -834,12 +879,14 @@ class many2many(_column):
"""
_classic_read = False
_classic_write = False
_prefetch = False
_type = 'many2many'
__slots__ = ['_obj', '_rel', '_id1', '_id2', '_limit', '_auto_join']
def __init__(self, obj, rel=None, id1=None, id2=None, string='unknown', limit=None, **args):
"""
"""
args['_prefetch'] = args.get('_prefetch', False)
_column.__init__(self, string=string, **args)
self._obj = obj
if rel and '.' in rel:
@ -849,6 +896,7 @@ class many2many(_column):
self._id1 = id1
self._id2 = id2
self._limit = limit
self._auto_join = False
def to_field_args(self):
args = super(many2many, self).to_field_args()
@ -1231,44 +1279,80 @@ class function(_column):
}
"""
_classic_read = False
_classic_write = False
_prefetch = False
_type = 'function'
_properties = True
# function fields are not copied by default
copy = False
__slots__ = [
'_type',
'_classic_read',
'_classic_write',
'_symbol_c',
'_symbol_f',
'_symbol_set',
'_symbol_get',
'_fnct',
'_arg',
'_fnct_inv',
'_fnct_inv_arg',
'_fnct_search',
'_multi',
'store',
'_digits',
'_digits_compute',
'selection',
'_obj',
]
@property
def digits(self):
if self._digits_compute:
with registry().cursor() as cr:
return self._digits_compute(cr)
else:
return self._digits
#
# multi: compute several fields in one call
#
def __init__(self, fnct, arg=None, fnct_inv=None, fnct_inv_arg=None, type='float', fnct_search=None, obj=None, store=False, multi=False, **args):
self._classic_read = False
self._classic_write = False
self._prefetch = False
self._symbol_c = '%s'
self._symbol_f = _symbol_set
self._symbol_set = (self._symbol_c, self._symbol_f)
self._symbol_get = None
# pop attributes that should not be assigned to self
self._digits = args.pop('digits', (16,2))
self._digits_compute = args.pop('digits_compute', None)
self._obj = args.pop('relation', obj)
# function fields are not copied by default
args['copy'] = args.get('copy', False)
_column.__init__(self, **args)
self._obj = obj
self._fnct = fnct
self._fnct_inv = fnct_inv
self._arg = arg
self._multi = multi
if 'relation' in args:
self._obj = args['relation']
self.digits = args.get('digits', (16,2))
self.digits_compute = args.get('digits_compute', None)
if callable(args.get('selection')):
from openerp import api
self.selection = api.expected(api.cr_uid_context, args['selection'])
self._fnct_inv_arg = fnct_inv_arg
if not fnct_inv:
self.readonly = 1
self._type = type
self._fnct = fnct
self._arg = arg
self._fnct_inv = fnct_inv
self._fnct_inv_arg = fnct_inv_arg
self._fnct_search = fnct_search
self.store = store
self._multi = multi
if not fnct_inv:
self.readonly = 1
if not fnct_search and not store:
self.selectable = False
if callable(args.get('selection')):
from openerp import api
self.selection = api.expected(api.cr_uid_context, args['selection'])
if store:
if self._type != 'many2one':
# m2o fields need to return tuples with name_get, not just foreign keys
@ -1283,6 +1367,10 @@ class function(_column):
self._symbol_c = char._symbol_c
self._symbol_f = lambda x: _symbol_set_char(self, x)
self._symbol_set = (self._symbol_c, self._symbol_f)
elif type == 'float':
self._symbol_c = float._symbol_c
self._symbol_f = lambda x: _symbol_set_float(self, x)
self._symbol_set = (self._symbol_c, self._symbol_f)
else:
type_class = globals().get(type)
if type_class is not None:
@ -1304,7 +1392,7 @@ class function(_column):
args = super(function, self).to_field_args()
args['store'] = bool(self.store)
if self._type in ('float',):
args['digits'] = self.digits_compute or self.digits
args['digits'] = self._digits_compute or self._digits
elif self._type in ('selection', 'reference'):
args['selection'] = self.selection
elif self._type in ('many2one', 'one2many', 'many2many'):
@ -1312,14 +1400,7 @@ class function(_column):
return args
def digits_change(self, cr):
if self._type == 'float':
if self.digits_compute:
self.digits = self.digits_compute(cr)
if self.digits:
precision, scale = self.digits
self._symbol_set = ('%s', lambda x: float_repr(float_round(__builtin__.float(x or 0.0),
precision_digits=scale),
precision_digits=scale))
pass
def search(self, cr, uid, obj, name, args, context=None):
if not self._fnct_search:
@ -1407,14 +1488,15 @@ class related(function):
'bar': fields.related('foo_id', 'frol', type='char', string='Frol of Foo'),
}
"""
__slots__ = ['arg', '_relations']
def _fnct_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None):
def _related_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None):
# assume self._arg = ('foo', 'bar', 'baz')
# domain = [(name, op, val)] => search [('foo.bar.baz', op, val)]
field = '.'.join(self._arg)
return map(lambda x: (field, x[1], x[2]), domain)
def _fnct_write(self, obj, cr, uid, ids, field_name, values, args, context=None):
def _related_write(self, obj, cr, uid, ids, field_name, values, args, context=None):
if isinstance(ids, (int, long)):
ids = [ids]
for instance in obj.browse(cr, uid, ids, context=context):
@ -1425,7 +1507,7 @@ class related(function):
# write on the last field of the target record
instance.write({self.arg[-1]: values})
def _fnct_read(self, obj, cr, uid, ids, field_name, args, context=None):
def _related_read(self, obj, cr, uid, ids, field_name, args, context=None):
res = {}
for record in obj.browse(cr, SUPERUSER_ID, ids, context=context):
value = record
@ -1452,13 +1534,14 @@ class related(function):
def __init__(self, *arg, **args):
self.arg = arg
self._relations = []
super(related, self).__init__(self._fnct_read, arg, self._fnct_write, fnct_inv_arg=arg, fnct_search=self._fnct_search, **args)
super(related, self).__init__(self._related_read, arg, self._related_write, fnct_inv_arg=arg, fnct_search=self._related_search, **args)
if self.store is True:
# TODO: improve here to change self.store = {...} according to related objects
pass
class sparse(function):
class sparse(function):
__slots__ = ['serialization_field']
def convert_value(self, obj, cr, uid, record, value, read_value, context=None):
"""
@ -1507,8 +1590,7 @@ class sparse(function):
return read_value
return value
def _fnct_write(self,obj,cr, uid, ids, field_name, value, args, context=None):
def _sparse_write(self,obj,cr, uid, ids, field_name, value, args, context=None):
if not type(ids) == list:
ids = [ids]
records = obj.browse(cr, uid, ids, context=context)
@ -1523,7 +1605,7 @@ class sparse(function):
obj.write(cr, uid, ids, {self.serialization_field: serialized}, context=context)
return True
def _fnct_read(self, obj, cr, uid, ids, field_names, args, context=None):
def _sparse_read(self, obj, cr, uid, ids, field_names, args, context=None):
results = {}
records = obj.browse(cr, uid, ids, context=context)
for record in records:
@ -1549,8 +1631,7 @@ class sparse(function):
def __init__(self, serialization_field, **kwargs):
self.serialization_field = serialization_field
super(sparse, self).__init__(self._fnct_read, fnct_inv=self._fnct_write, multi='__sparse_multi', **kwargs)
super(sparse, self).__init__(self._sparse_read, fnct_inv=self._sparse_write, multi='__sparse_multi', **kwargs)
# ---------------------------------------------------------
@ -1558,19 +1639,21 @@ class sparse(function):
# ---------------------------------------------------------
class dummy(function):
def _fnct_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None):
__slots__ = ['arg', '_relations']
def _dummy_search(self, tobj, cr, uid, obj=None, name=None, domain=None, context=None):
return []
def _fnct_write(self, obj, cr, uid, ids, field_name, values, args, context=None):
def _dummy_write(self, obj, cr, uid, ids, field_name, values, args, context=None):
return False
def _fnct_read(self, obj, cr, uid, ids, field_name, args, context=None):
def _dummy_read(self, obj, cr, uid, ids, field_name, args, context=None):
return {}
def __init__(self, *arg, **args):
self.arg = arg
self._relations = []
super(dummy, self).__init__(self._fnct_read, arg, self._fnct_write, fnct_inv_arg=arg, fnct_search=self._fnct_search, **args)
super(dummy, self).__init__(self._dummy_read, arg, self._dummy_write, fnct_inv_arg=arg, fnct_search=self._dummy_search, **args)
# ---------------------------------------------------------
# Serialized fields
@ -1581,42 +1664,46 @@ class serialized(_column):
Note: only plain components allowed.
"""
_type = 'serialized'
__slots__ = []
def _symbol_set_struct(val):
return simplejson.dumps(val)
def _symbol_get_struct(self, val):
return simplejson.loads(val or '{}')
_prefetch = False
_type = 'serialized'
_symbol_c = '%s'
_symbol_f = _symbol_set_struct
_symbol_set = (_symbol_c, _symbol_f)
_symbol_get = _symbol_get_struct
def __init__(self, *args, **kwargs):
kwargs['_prefetch'] = kwargs.get('_prefetch', False)
super(serialized, self).__init__(*args, **kwargs)
# TODO: review completly this class for speed improvement
class property(function):
__slots__ = []
def to_field_args(self):
args = super(property, self).to_field_args()
args['company_dependent'] = True
return args
def _fnct_search(self, tobj, cr, uid, obj, name, domain, context=None):
def _property_search(self, tobj, cr, uid, obj, name, domain, context=None):
ir_property = obj.pool['ir.property']
result = []
for field, operator, value in domain:
result += ir_property.search_multi(cr, uid, name, tobj._name, operator, value, context=context)
return result
def _fnct_write(self, obj, cr, uid, id, prop_name, value, obj_dest, context=None):
def _property_write(self, obj, cr, uid, id, prop_name, value, obj_dest, context=None):
ir_property = obj.pool['ir.property']
ir_property.set_multi(cr, uid, prop_name, obj._name, {id: value}, context=context)
return True
def _fnct_read(self, obj, cr, uid, ids, prop_names, obj_dest, context=None):
def _property_read(self, obj, cr, uid, ids, prop_names, obj_dest, context=None):
ir_property = obj.pool['ir.property']
res = {id: {} for id in ids}
@ -1645,9 +1732,9 @@ class property(function):
args = dict(args)
args['obj'] = args.pop('relation', '') or args.get('obj', '')
super(property, self).__init__(
fnct=self._fnct_read,
fnct_inv=self._fnct_write,
fnct_search=self._fnct_search,
fnct=self._property_read,
fnct_inv=self._property_write,
fnct_search=self._property_search,
multi='properties',
**args
)
@ -1681,6 +1768,8 @@ class column_info(object):
contains it i.e in case of multilevel inheritance, ``None`` for
local columns.
"""
__slots__ = ['name', 'column', 'parent_model', 'parent_column', 'original_parent']
def __init__(self, name, column, parent_model=None, parent_column=None, original_parent=None):
self.name = name
self.column = column

8
openerp/service/server.py
View File

@ -38,7 +38,7 @@ import openerp
from openerp.modules.registry import RegistryManager
from openerp.release import nt_service_name
import openerp.tools.config as config
from openerp.tools.misc import stripped_sys_argv, dumpstacks
from openerp.tools import stripped_sys_argv, dumpstacks, log_ormcache_stats
_logger = logging.getLogger(__name__)
@ -296,6 +296,7 @@ class ThreadedServer(CommonServer):
signal.signal(signal.SIGCHLD, self.signal_handler)
signal.signal(signal.SIGHUP, self.signal_handler)
signal.signal(signal.SIGQUIT, dumpstacks)
signal.signal(signal.SIGUSR1, log_ormcache_stats)
elif os.name == 'nt':
import win32api
win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1)
@ -389,6 +390,7 @@ class GeventServer(CommonServer):
if os.name == 'posix':
signal.signal(signal.SIGQUIT, dumpstacks)
signal.signal(signal.SIGUSR1, log_ormcache_stats)
gevent.spawn(self.watch_parent)
self.httpd = WSGIServer((self.interface, self.port), self.app)
@ -510,6 +512,9 @@ class PreforkServer(CommonServer):
elif sig == signal.SIGQUIT:
# dump stacks on kill -3
self.dumpstacks()
elif sig == signal.SIGUSR1:
# log ormcache stats on kill -SIGUSR1
log_ormcache_stats()
elif sig == signal.SIGTTIN:
# increase number of workers
self.population += 1
@ -586,6 +591,7 @@ class PreforkServer(CommonServer):
signal.signal(signal.SIGTTIN, self.signal_handler)
signal.signal(signal.SIGTTOU, self.signal_handler)
signal.signal(signal.SIGQUIT, dumpstacks)
signal.signal(signal.SIGUSR1, log_ormcache_stats)
# listen to socket
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

93
openerp/tools/cache.py
View File

@ -21,13 +21,29 @@
# decorator makes wrappers that have the same API as their wrapped function;
# this is important for the openerp.api.guess() that relies on signatures
from collections import defaultdict
from decorator import decorator
from inspect import getargspec
import lru
import logging
logger = logging.getLogger(__name__)
_logger = logging.getLogger(__name__)
class ormcache_counter(object):
""" Statistic counters for cache entries. """
__slots__ = ['hit', 'miss', 'err']
def __init__(self):
self.hit = 0
self.miss = 0
self.err = 0
@property
def ratio(self):
return 100.0 * self.hit / (self.hit + self.miss or 1)
# statistic counters dictionary, maps (dbname, modelname, method) to counter
STAT = defaultdict(ormcache_counter)
class ormcache(object):
@ -35,10 +51,6 @@ class ormcache(object):
def __init__(self, skiparg=2, size=8192, multi=None, timeout=None):
self.skiparg = skiparg
self.size = size
self.stat_miss = 0
self.stat_hit = 0
self.stat_err = 0
def __call__(self, method):
self.method = method
@ -46,42 +58,32 @@ class ormcache(object):
lookup.clear_cache = self.clear
return lookup
def stat(self):
return "lookup-stats hit=%s miss=%s err=%s ratio=%.1f" % \
(self.stat_hit, self.stat_miss, self.stat_err,
(100*float(self.stat_hit))/(self.stat_miss+self.stat_hit))
def lru(self, model):
ormcache = model._ormcache
try:
d = ormcache[self.method]
except KeyError:
d = ormcache[self.method] = lru.LRU(self.size)
return d
return model.pool.cache, (model.pool.db_name, model._name, self.method)
def lookup(self, method, *args, **kwargs):
d = self.lru(args[0])
key = args[self.skiparg:]
d, key0 = self.lru(args[0])
key = key0 + args[self.skiparg:]
try:
r = d[key]
self.stat_hit += 1
STAT[key0].hit += 1
return r
except KeyError:
self.stat_miss += 1
STAT[key0].miss += 1
value = d[key] = self.method(*args, **kwargs)
return value
except TypeError:
self.stat_err += 1
STAT[key0].err += 1
return self.method(*args, **kwargs)
def clear(self, model, *args):
""" Remove *args entry from the cache or all keys if *args is undefined """
d = self.lru(model)
d, key0 = self.lru(model)
if args:
logger.warn("ormcache.clear arguments are deprecated and ignored "
"(while clearing caches on (%s).%s)",
model._name, self.method.__name__)
d.clear()
_logger.warn("ormcache.clear arguments are deprecated and ignored "
"(while clearing caches on (%s).%s)",
model._name, self.method.__name__)
d.clear_prefix(key0)
model.pool._any_cache_cleared = True
@ -97,7 +99,7 @@ class ormcache_context(ormcache):
return super(ormcache_context, self).__call__(method)
def lookup(self, method, *args, **kwargs):
d = self.lru(args[0])
d, key0 = self.lru(args[0])
# Note. The decorator() wrapper (used in __call__ above) will resolve
# arguments, and pass them positionally to lookup(). This is why context
@ -109,17 +111,17 @@ class ormcache_context(ormcache):
ckey = [(k, context[k]) for k in self.accepted_keys if k in context]
# Beware: do not take the context from args!
key = args[self.skiparg:self.context_pos] + tuple(ckey)
key = key0 + args[self.skiparg:self.context_pos] + tuple(ckey)
try:
r = d[key]
self.stat_hit += 1
STAT[key0].hit += 1
return r
except KeyError:
self.stat_miss += 1
STAT[key0].miss += 1
value = d[key] = self.method(*args, **kwargs)
return value
except TypeError:
self.stat_err += 1
STAT[key0].err += 1
return self.method(*args, **kwargs)
@ -130,8 +132,8 @@ class ormcache_multi(ormcache):
self.multi = multi
def lookup(self, method, *args, **kwargs):
d = self.lru(args[0])
base_key = args[self.skiparg:self.multi] + args[self.multi+1:]
d, key0 = self.lru(args[0])
base_key = key0 + args[self.skiparg:self.multi] + args[self.multi+1:]
ids = args[self.multi]
result = {}
missed = []
@ -141,9 +143,9 @@ class ormcache_multi(ormcache):
key = base_key + (i,)
try:
result[i] = d[key]
self.stat_hit += 1
STAT[key0].hit += 1
except Exception:
self.stat_miss += 1
STAT[key0].miss += 1
missed.append(i)
if missed:
@ -173,6 +175,23 @@ class dummy_cache(object):
pass
def log_ormcache_stats(sig=None, frame=None):
""" Log statistics of ormcache usage by database, model, and method. """
from openerp.modules.registry import RegistryManager
import threading
me = threading.currentThread()
entries = defaultdict(int)
for key in RegistryManager.cache.iterkeys():
entries[key[:3]] += 1
for key, count in sorted(entries.items()):
dbname, model_name, method = key
me.dbname = dbname
stat = STAT[key]
_logger.info("%6d entries, %6d hit, %6d miss, %6d err, %4.1f%% ratio, for %s.%s",
count, stat.hit, stat.miss, stat.err, stat.ratio, model_name, method.__name__)
# For backward compatibility
cache = ormcache

8
openerp/tools/lru.py
View File

@ -119,4 +119,12 @@ class LRU(object):
self.first = None
self.last = None
@synchronized()
def clear_prefix(self, prefix):
""" Remove from `self` all the items with the given `prefix`. """
n = len(prefix)
for key in self.keys():
if key[:n] == prefix:
del self[key]
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: