[IMP] mail.alias: work-in-progress: cleanup actual email processing

bzr revid: odo@openerp.com-20120807180412-u2gdsjg579ak7zxp
This commit is contained in:
Olivier Dony 2012-08-07 20:04:12 +02:00
parent 60722930cd
commit 86836d67dd
11 changed files with 179 additions and 184 deletions

View File

@ -88,9 +88,10 @@
<separator string="Emails Integration" attrs="{'invisible': [('module_crm','=',False)]}"/>
<p attrs="{'invisible': [('module_crm','=',False)]}">
OpenERP allows to automatically create leads (or others documents)
from incoming emails. You can synchronize emails with OpenERP
by connecting the mail gateway to your mail server or by manually
pressing buttons in our mail clients.
from incoming emails. You can automatically synchronize emails with OpenERP
using regular POP/IMAP accounts, using a direct email integration script for your
email server, or by manually pushing emails to OpenERP using specific
plugins for your preferred email application.
</p>
<group name="On Mail Client" attrs="{'invisible': [('module_crm','=',False)]}">
<label for="id" string="On Mail Client"/>

View File

@ -26,17 +26,10 @@
<record model="crm.case.channel">
<field name="name">email</field>
</record>
<record id="mail_alias_sales_department" model="mail.alias">
<field name="alias_name">sales</field>
<field name="alias_model_id" ref="model_crm_lead"/>
<field name="alias_user_id" ref="base.user_root"/>
<field name="alias_defaults">{'type':'lead'}</field>
</record>
<record model="crm.case.section" id="section_sales_department">
<field name="name">Sales Department</field>
<field name="code">Sales</field>
<field name="alias_id" ref="mail_alias_sales_department"/>
</record>
<!-- Payment Mode -->
@ -70,5 +63,13 @@
To manage quotations and sale orders, install the module "Sales Management".</value>
</function>
<record model="mail.group" name="default_sales_alias">
<field name="alias_name">sales</field>
<field name="alias_model_id" ref="model_crm_lead"/>
<field name="alias_user_id" ref="base.user_root"/>
<field name="alias_defaults">{'type':'lead'}</field>
</record>
</data>
</openerp>

View File

@ -1,14 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<openerp>
<data noupdate="1">
<record id="mail_alias_marketing_department" model="mail.alias">
<field name="alias_name">info</field>
<field name="alias_model_id" ref="model_crm_lead"/>
<field name="alias_user_id" ref="base.user_root"/>
<field name="alias_defaults">{'type':'lead'}</field>
</record>
</data>
<data>
<record id="base.user_demo" model="res.users">
<field name="groups_id" eval="[(4,ref('base.group_sale_salesman'))]"/>
@ -20,12 +11,11 @@
<field name="user_id" ref="base.user_demo"/>
<field name="res_id" ref="base_calendar.menu_crm_meeting"/>
</record>
<record model="crm.case.section" id="section_sales_marketing_department">
<field name="name">Sales Marketing Department</field>
<field name="code">Sales Marketing</field>
<field name="parent_id" ref="crm.section_sales_department"></field>
<field name="alias_id" ref="mail_alias_marketing_department"/>
</record>
<record model="crm.segmentation" id="crm_segmentation0">

View File

@ -72,8 +72,6 @@
</record>
<record model="crm.case.section" id="section_sales_department">
<field name="name">Sales Department</field>
<field name="code">Sales</field>
<field name="stage_ids" eval="[ (4, ref('stage_lead1')), (4, ref('stage_lead2')),
(4, ref('stage_lead3')), (4, ref('stage_lead4')),
(4, ref('stage_lead5')), (4, ref('stage_lead6')),

View File

@ -22,9 +22,9 @@
</group>
</group>
<div name="config_other" version="7.0" position="inside">
<separator string="Import and Synchronize Data from an Other Application"/>
<separator string="Importing and Synchronizing with External Systems"/>
<group>
<label for="id" string="Import From"/>
<label for="id" string="Import"/>
<div>
<div>
<field name="module_import_sugarcrm"/>
@ -33,7 +33,7 @@
</div>
</group>
<group>
<label for="id" string="Synchronize with"/>
<label for="id" string="Synchronize"/>
<div>
<div>
<field name="module_import_google"/>
@ -43,6 +43,7 @@
<field name="module_crm_caldav"/>
<label for="module_crm_caldav"/>
</div>
</div>
</group>
</div>

View File

@ -189,14 +189,11 @@ openerp_mailgate.py -u %(uid)d -p PASSWORD -o %(model)s -d %(dbname)s --host=HOS
result, data = imap_server.search(None, '(UNSEEN)')
for num in data[0].split():
result, data = imap_server.fetch(num, '(RFC822)')
if server.object_id:
res_id = mail_thread.message_process(cr, uid, server.object_id.model,
data[0][1],
save_original=server.original,
strip_attachments=(not server.attach),
context=context)
else:
res_id = mail_thread.message_catchall(cr, uid, data[0][1])
res_id = mail_thread.message_process(cr, uid, server.object_id.model,
data[0][1],
save_original=server.original,
strip_attachments=(not server.attach),
context=context)
if res_id and server.action_id:
action_pool.run(cr, uid, [server.action_id.id], {'active_id': res_id, 'active_ids':[res_id]})
imap_server.store(num, '+FLAGS', '\\Seen')
@ -217,14 +214,11 @@ openerp_mailgate.py -u %(uid)d -p PASSWORD -o %(model)s -d %(dbname)s --host=HOS
for num in range(1, numMsgs + 1):
(header, msges, octets) = pop_server.retr(num)
msg = '\n'.join(msges)
if server.object_id:
res_id = mail_thread.message_process(cr, uid, server.object_id.model,
msg,
save_original=server.original,
strip_attachments=(not server.attach),
context=context)
else:
res_id = mail_thread.message_catchall(cr, uid, data[0][1])
res_id = mail_thread.message_process(cr, uid, server.object_id.model,
msg,
save_original=server.original,
strip_attachments=(not server.attach),
context=context)
if res_id and server.action_id:
action_pool.run(cr, uid, [server.action_id.id], {'active_id': res_id, 'active_ids':[res_id]})
pop_server.dele(num)

View File

@ -458,7 +458,6 @@ You can automatically create application records from an email gateway, that you
<field name="alias_name">jobs</field>
<field name="alias_model_id" ref="model_hr_applicant"/>
<field name="alias_user_id" ref="base.user_root"/>
<field name="alias_defaults">{}</field>
</record>
</data>

View File

@ -20,9 +20,19 @@
##############################################################################
import re
import unicodedata
from openerp.osv import fields, osv
from openerp.tools import ustr
# Inspired by http://stackoverflow.com/questions/517923
def remove_accents(input_str):
"""Suboptimal-but-better-than-nothing way to replace accented
latin letters by an ASCII equivalent. Will obviously change the
meaning of input_str and work only for some cases"""
input_str = ustr(input_str)
nkfd_form = unicodedata.normalize('NFKD', input_str)
return u''.join([c for c in nkfd_form if not unicodedata.combining(c)])
class mail_alias(osv.Model):
"""A Mail Alias is a mapping of an email address with a given OpenERP Document
@ -40,6 +50,7 @@ class mail_alias(osv.Model):
_name = 'mail.alias'
_description = "Email Aliases"
_rec_name = 'alias_name'
_order = 'alias_model_id, alias_name'
def _get_alias_domain(self, cr, uid, ids, name, args, context=None):
ir_config_parameter = self.pool.get("ir.config_parameter")
@ -55,10 +66,9 @@ class mail_alias(osv.Model):
"corresponds. Any incoming email that does not reply to an "
"existing record will cause the creation of a new record "
"of this model (e.g. a Project Task)",
# only allow selecting mail_thread models!
#TODO kept doamin temporarily in comment, need to redefine domain
#domain="[('field_id', 'in', 'message_ids')]"
),
# hack to only allow selecting mail_thread models (we might
# (have a few false positives, though)
domain="[('field_id.name', '=', 'message_ids')]"),
'alias_user_id': fields.many2one('res.users', 'Owner',
help="The owner of records created upon receiving emails on this alias. "
"If this field is not set the system will attempt to find the right owner "
@ -76,7 +86,10 @@ class mail_alias(osv.Model):
_defaults = {
'alias_defaults': '{}',
'alias_user_id': lambda self,cr,uid, context: uid
'alias_user_id': lambda self,cr,uid,context: uid,
# looks better when creating new aliases - even if the field is informative only
'alias_domain': lambda self,cr,uid,context: self._get_alias_domain(cr,1,[1],None,None)[1]
}
_sql_constraints = [
@ -122,12 +135,10 @@ class mail_alias(osv.Model):
make it unique, and the ``alias_model_id`` value will set to the
model ID of the ``model_name`` value, if provided,
"""
alias_name = re.sub(r'\W+', '_', vals['alias_name']).lower()
alias_name = re.sub(r'[^\w+]', '_', remove_accents(vals['alias_name'])).lower()
alias_name = self._find_unique(cr, uid, alias_name, context=context)
vals['alias_name'] = alias_name
if model_name:
model_id = self.pool.get('ir.model').search(cr, uid, [('model', '=', model_name)], context=context)[0]
vals['alias_model_id'] = model_id
return self.create(cr, uid, vals, context=context)

View File

@ -11,14 +11,12 @@
<form string="Alias" version="7.0">
<sheet>
<label for="alias_name" class="oe_edit_only"/>
<h2><field name="alias_name" class="oe_inline"/>@<field name="alias_domain" class="oe_inline"/></h2>
<group colspan="4" col="4">
<field name="alias_model_id" readonly="1"/>
<field name="alias_user_id" readonly="1"/>
<field name="alias_force_thread_id" readonly="1"/>
<newline/>
<separator string="Default Values" colspan="4"/>
<field name="alias_defaults" colspan="4" nolabel="1" readonly="1"/>
<h2><field name="alias_name" class="oe_inline"/>@<field name="alias_domain" class="oe_inline"/></h2>
<group>
<field name="alias_model_id"/>
<field name="alias_user_id"/>
<field name="alias_force_thread_id"/>
<field name="alias_defaults"/>
</group>
</sheet>
</form>

View File

@ -560,79 +560,109 @@ class mail_thread(osv.Model):
msgs = msg_obj.read(cr, uid, msg_ids, context=context)
return msgs
def _get_user(self, cr, uid, alias, context):
"""
param alias: browse record of alias.
return: int user_id.
"""
user_obj = self.pool.get('res.user')
user_id = 1
if alias.alias_user_id:
user_id = alias.alias_user_id.id
#if user_id not defined in the alias then search related user using name of Email sender
else:
from_email = msg.get('from')
user_ids = user_obj.search(cr, uid, [('name','=',from_email)], context)
if user_ids:
user_id = user_obj.browse(cr, uid, user_ids[0], context).id
return user_id
def message_catchall(self, cr, uid, message, context=None):
"""
Process incoming mail and call messsage_process using details of the mail.alias model
else raise Exception so that mailgate script will reject the mail and
send notification mail sender that this mailbox does not exist so your mail have been rejected.
"""
mail_alias = self.pool.get('mail.alias')
mail_message = self.pool.get('mail.message')
if isinstance(message, xmlrpclib.Binary):
message = str(message.data)
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
msg = mail_message.parse_message(msg_txt)
alias_name = msg.get('to').split("@")[0] # @@@@
alias_ids = mail_alias.search(cr, uid, [('alias_name','=',alias_name)])
#if alias found then call message_process method. # @@@@
if alias_ids:
alias_id = mail_alias.browse(cr, uid, alias_ids[0], context)
user_id = self._get_user( cr, uid, alias_id, context)
alias_defaults = dict(eval(alias_id.alias_defaults or {}))
self.message_process(cr, user_id, alias_id.alias_model_id.model, message,
custom_values=alias_defaults,
thread_id=alias_id.alias_force_thread_id or False,
context=context)
else:
#if Mail box for the intended Mail Alias then give logger warning
_logger.warning("No Mail Alias Found for the name '%s'."%(alias_name))
raise # @@@@
return True
def _message_find_user_id(self, cr, uid, message, context=None):
from_local_part = to_email(decode(message.get('From')))[0]
user_ids = self.pool.get('res.users').search(cr, uid, [('login', '=', from_local_part)], context=context)
return user_ids[0] if user_ids else uid
#------------------------------------------------------
# Mail gateway
#------------------------------------------------------
# message_process will call either message_new or message_update.
def message_route(self, cr, uid, message, model=None, thread_id=None,
custom_values=None, context=None):
"""Attempt to figure out the correct target model, thread_id,
custom_values and user_id to use for an incoming message.
The following heuristics are used, in this order:
1. If the message replies to an existing thread_id, and
properly contains the thread model in the 'In-Reply-To'
header, use this model/thread_id pair, and ignore
custom_value (not needed as no creation will take place)
2. Look for a mail.alias entry matching the message
recipient, and use the corresponding model, thread_id,
custom_values and user_id.
3. Fallback to the ``model``, ``thread_id`` and ``custom_values``
provided.
4. If all the above fails, raise an exception.
:param string message: an email.message instance
:param string model: the fallback model to use if the message
does not match any of the currently configured mail aliases
(may be None if a matching alias is supposed to be present)
:type dict custom_values: optional dictionary of default field values
to pass to ``message_new`` if a new record needs to be created.
Ignored if the thread record already exists, and also if a
matching mail.alias was found (aliases define their own defaults)
:param int thread_id: optional ID of the record/thread from ``model``
to which this mail should be attached. Only used if the message
does not reply to an existing thread and does not match any mail alias.
:return: model, thread_id, custom_values, user_id
"""
assert isinstance(message, email.Message), 'message must be an email.Message at this point'
# 1. Verify if this is a reply to an existing thread
references = message.get('References') or message.get('In-Reply-To')
ref_match = tools.reference_re.search(references)
if ref_match:
thread_id = int(ref_match.group(1))
model = ref_match.group(2) or model
model_pool = self.pool.get(model)
if thread_id and model and model_pool and model_pool.exists(cr, uid, thread_id) \
and hasattr(model_pool, 'message_update'):
return model, thread_id, custom_values, uid
# 2. Look for a matching mail.alias entry
# Delivered-To is a safe bet in most modern MTAs, but we have to fallback on To + Cc values
# for all the odd MTAs out there, as there is no standard header for the envelope's `rcpt_to` value.
rcpt_to = message.get_all('Delivered-To', []) or (message.get_all('To', []) + message.get_all('Cc', []))
local_parts = [e.split('@')[0] for e in to_email(u','.join(decode(rcpt_to)))]
if local_parts:
mail_alias = self.pool.get('mail.alias')
alias_ids = mail_alias.search(cr, uid, [('alias_name', 'in', local_parts)])
if len(alias_ids) > 1:
_logger.warning('Multiple mail.aliases match for mail with Message-Id %s, keeping first one only: %s',
message.get('Message-Id'), alias_ids)
alias = mail_alias.browse(cr, uid, alias_ids[0], context=context)
user_id = alias.alias_user_id.id
if not user_id:
user_id = self._message_find_user_id(cr, uid, message, context=context)
return alias.alias_model_id.model, alias.alias_model_id.alias_force_thread_id, \
alias.alias_defaults, user_id
# 3. Fallback to the provided parameters, if they work
model_pool = self.pool.get(model)
assert thread_id and hasattr(model_pool, 'message_update') or hasattr(model_pool, 'message_new'), \
"No possible route found for incoming message with Message-Id %s. " \
"Create an appropriate mail.alias or force the destination model."
return model, thread_id, custom_values, uid
def message_process(self, cr, uid, model, message, custom_values=None,
save_original=False, strip_attachments=False,
thread_id=None, context=None):
"""Process an incoming RFC2822 email message related to the
given thread model, relying on ``mail.message.parse()``
for the parsing operation, and then calling ``message_new``
(if the thread record did not exist) or ``message_update``
(if it did), then calling ``message_forward`` to automatically
notify other people that should receive this message.
"""Process an incoming RFC2822 email message, relying on
``mail.message.parse()`` for the parsing operation,
and ``message_route()`` to figure out the target model.
Once the target model is known, its ``message_new`` method
is called with the new message (if the thread record did not exist)
or its ``message_update`` method (if it did). Finally,
``message_forward`` is called to automatically notify other
people that should receive this message.
:param string model: the thread model for which a new message
must be processed
:param message: source of the RFC2822 mail
:param string model: the fallback model to use if the message
does not match any of the currently configured mail aliases
(may be None if a matching alias is supposed to be present)
:param message: source of the RFC2822 message
:type message: string or xmlrpclib.Binary
:type dict custom_values: optional dictionary of field values
to pass to ``message_new`` if a new
record needs to be created. Ignored
if the thread record already exists.
to pass to ``message_new`` if a new record needs to be created.
Ignored if the thread record already exists, and also if a
matching mail.alias was found (aliases define their own defaults)
:param bool save_original: whether to keep a copy of the original
email source attached to the message after it is imported.
:param bool strip_attachments: whether to strip all attachments
@ -642,64 +672,39 @@ class mail_thread(osv.Model):
overrides the automatic detection based on the message
headers.
"""
if context is None: context = {}
# extract message bytes - we are forced to pass the message as binary because
# we don't know its encoding until we parse its headers and hence can't
# convert it to utf-8 for transport between the mailgate script and here.
if isinstance(message, xmlrpclib.Binary):
message = str(message.data)
if context is None: context = {}
mail_message = self.pool.get('mail.message')
model_pool = self.pool.get(model)
if self._name != model:
context.update({'thread_model': model})
# Parse Message
# Warning: message_from_string doesn't always work correctly on unicode,
# we must use utf-8 strings here :-(
if isinstance(message, unicode):
message = message.encode('utf-8')
msg_txt = email.message_from_string(message)
msg = mail_message.parse_message(msg_txt, save_original=save_original, context=context)
# update state
msg['state'] = 'received'
model, thread_id, custom_values, user_id = self.message_route(cr, uid, msg_txt, model,
thread_id, custom_values,
context=context)
if self._name != model:
context.update({'thread_model': model})
msg = self.pool.get('mail.message').parse_message(msg_txt, save_original=save_original, context=context)
msg['state'] = 'received'
if strip_attachments and 'attachments' in msg:
del msg['attachments']
# Create New Record into particular model
def create_record(msg):
if hasattr(model_pool, 'message_new'):
return model_pool.message_new(cr, uid, msg,
custom_values,
context=context)
if not thread_id and (msg.get('references') or msg.get('in-reply-to')):
references = msg.get('references') or msg.get('in-reply-to')
if '\r\n' in references:
references = references.split('\r\n')
else:
references = references.split(' ')
for ref in references:
ref = ref.strip()
thread_id = tools.reference_re.search(ref)
if not thread_id:
thread_id = tools.res_re.search(msg['subject'])
if thread_id:
thread_id = int(thread_id.group(1))
if not model_pool.exists(cr, uid, thread_id) or \
not hasattr(model_pool, 'message_update'):
# referenced thread not found or not updatable,
# -> create a new one
thread_id = False
if not thread_id:
thread_id = create_record(msg)
model_pool = self.pool.get(model)
assert thread_id and hasattr(model_pool, 'message_update') or hasattr(model_pool, 'message_new'), \
"Undeliverable mail with Message-Id %s, model %s does not accept incoming emails" % \
(msg['message-id'], model)
if thread_id and hasattr(model_pool, 'message_update'):
model_pool.message_update(cr, user_id, [thread_id], msg, {}, context=context)
else:
model_pool.message_update(cr, uid, [thread_id], msg, {}, context=context)
# To forward the email to other followers
thread_id = model_pool.message_new(cr, user_id, msg, custom_values, context=context)
# Forward the email to other followers
self.message_forward(cr, uid, model, [thread_id], msg_txt, context=context)
# Set as Unread
model_pool.message_mark_as_unread(cr, uid, [thread_id], context=context)
return thread_id

View File

@ -106,15 +106,15 @@ class EmailParser(object):
self.email_default = email_default
def parse(self, method, message, custom_values=None, save_original=None):
def parse(self, message, custom_values=None, save_original=None):
# pass message as bytes because we don't know its encoding until we parse its headers
# and hence can't convert it to utf-8 for transport
res_id = self.rpc('mail.thread',
method,
self.model,
xmlrpclib.Binary(message),
custom_values or {},
save_original or False)
return self.rpc('mail.thread',
'message_process',
self.model,
xmlrpclib.Binary(message),
custom_values or {},
save_original or False)
def configure_parser():
parser = optparse.OptionParser(usage='usage: %prog [options]', version='%prog v1.1')
@ -123,32 +123,32 @@ def configure_parser():
"with the OpenERP server for case management in the CRM module.")
parser.add_option_group(group)
parser.add_option("-u", "--user", dest="userid",
help="ID of the user in OpenERP",
help="OpenERP user id to connect with",
default=config.OPENERP_DEFAULT_USER_ID, type='int')
parser.add_option("-p", "--password", dest="password",
help="Password of the user in OpenERP",
help="OpenERP user password",
default=config.OPENERP_DEFAULT_PASSWORD)
parser.add_option("-o", "--model", dest="model",
help="Name or ID of crm model",
help="Name or ID of destination model",
default="crm.lead")
parser.add_option("-m", "--default", dest="default",
help="Default eMail in case of any trouble.",
help="Admin email for error notifications.",
default=None)
parser.add_option("-d", "--dbname", dest="dbname",
help="Database name (default: %default)",
help="OpenERP database name (default: %default)",
default=config.OPENERP_DEFAULT_DATABASE)
parser.add_option("--host", dest="host",
help="Hostname of the OpenERP Server",
help="OpenERP Server hostname",
default=config.OPENERP_HOSTNAME)
parser.add_option("--port", dest="port",
help="Port of the OpenERP Server",
help="OpenERP Server XML-RPC port number",
default=config.OPENERP_PORT)
parser.add_option("--custom-values", dest="custom_values",
help="Add Custom Values to the object",
help="Dictionary of extra values to pass when creating records",
default=None)
parser.add_option("-s", dest="save_original",
action="store_true",
help="Attach a copy of original email to the message entry",
help="Keep a full copy of the email source attached to each message",
default=False)
return parser
@ -160,7 +160,6 @@ def main():
parser = configure_parser()
(options, args) = parser.parse_args()
method = "message_process"
email_parser = EmailParser(options.userid,
options.password,
options.dbname,
@ -170,8 +169,6 @@ def main():
email_default= options.default)
msg_txt = sys.stdin.read()
custom_values = {}
if not options.model:
method = "message_catchall"
try:
custom_values = dict(eval(options.custom_values or "{}" ))
except:
@ -179,7 +176,7 @@ def main():
traceback.print_exc()
try:
email_parser.parse(method, msg_txt, custom_values, options.save_original or False)
email_parser.parse(msg_txt, custom_values, options.save_original or False)
except Exception:
msg = '\n'.join([
'parameters',