Skip to content

Commit

Permalink
Merge pull request #155 from zevaverbach/vendors
Browse files Browse the repository at this point in the history
fix substring search of subjects in Gmail, add support for some Gmail extended IMAP
  • Loading branch information
martinrusev committed Oct 19, 2018
2 parents 782d397 + 12a8cd9 commit 8d7d133
Show file tree
Hide file tree
Showing 11 changed files with 118 additions and 93 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,6 @@ example.py

# PyCharm
.idea/

# Mac
.DS_Store
3 changes: 3 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"python.pythonPath": "/usr/local/bin/python3"
}
6 changes: 5 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,15 @@ Usage
inbox_messages_subject_christmas = imbox.messages(subject='Christmas')
# Messages whose UID is greater than 1050
inbox_messages_subject_christmas = imbox.messages(uid__range='1050:*')
inbox_messages_uids_greater_than_1050 = imbox.messages(uid__range='1050:*')
# Messages from a specific folder
messages_in_folder_social = imbox.messages(folder='Social')
# Some of Gmail's IMAP Extensions are supported (label and raw):
all_messages_with_an_attachment_from_martin = imbox.messages(folder='all', raw='from:martin@amon.cx has:attachment')
all_messages_labeled_finance = imbox.messages(folder='all', label='finance')
for uid, message in all_inbox_messages:
# Every message is an object with the following keys
Expand Down
20 changes: 13 additions & 7 deletions imbox/imbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@ def __init__(self, hostname, username=None, password=None, ssl=True,
self.vendor = vendor or hostname_vendorname_dict.get(self.hostname)

if self.vendor is not None:
self.authentication_error_message = name_authentication_string_dict.get(self.vendor)
self.authentication_error_message = name_authentication_string_dict.get(
self.vendor)

try:
self.connection = self.server.connect(username, password)
except imaplib.IMAP4.error as e:
if self.authentication_error_message is None:
raise
raise imaplib.IMAP4.error(self.authentication_error_message + '\n' + str(e))
raise imaplib.IMAP4.error(
self.authentication_error_message + '\n' + str(e))

logger.info("Connected to IMAP Server with user {username} on {hostname}{ssl}".format(
hostname=hostname, username=username, ssl=(" over SSL" if ssl or starttls else "")))
Expand All @@ -61,16 +63,18 @@ def mark_flag(self, uid):
self.connection.uid('STORE', uid, '+FLAGS', '(\\Flagged)')

def delete(self, uid):
logger.info("Mark UID {} with \\Deleted FLAG and expunge.".format(int(uid)))
self.connection.uid('STORE', uid, '+FLAGS', '(\\Deleted)')
logger.info(
"Mark UID {} with \\Deleted FLAG and expunge.".format(int(uid)))
self.connection.expunge()

def copy(self, uid, destination_folder):
logger.info("Copy UID {} to {} folder".format(int(uid), str(destination_folder)))
logger.info("Copy UID {} to {} folder".format(
int(uid), str(destination_folder)))
return self.connection.uid('COPY', uid, destination_folder)

def move(self, uid, destination_folder):
logger.info("Move UID {} to {} folder".format(int(uid), str(destination_folder)))
logger.info("Move UID {} to {} folder".format(
int(uid), str(destination_folder)))
if self.copy(uid, destination_folder):
self.delete(uid)

Expand All @@ -83,8 +87,10 @@ def messages(self, **kwargs):
messages_class = GmailMessages

if folder:
self.connection.select(messages_class.folder_lookup.get((folder.lower())) or folder)
self.connection.select(
messages_class.FOLDER_LOOKUP.get((folder.lower())) or folder)
msg = " from folder '{}'".format(folder)
del kwargs['folder']
else:
msg = " from inbox"

Expand Down
25 changes: 20 additions & 5 deletions imbox/messages.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,29 @@
from imbox.parser import fetch_email_by_uid
import datetime
import logging

from imbox.query import build_search_query
from imbox.parser import fetch_email_by_uid

import logging

logger = logging.getLogger(__name__)


class Messages:

folder_lookup = {}
IMAP_ATTRIBUTE_LOOKUP = {
'unread': '(UNSEEN)',
'flagged': '(FLAGGED)',
'unflagged': '(UNFLAGGED)',
'sent_from': '(FROM "{}")',
'sent_to': '(TO "{}")',
'date__gt': '(SINCE "{}")',
'date__lt': '(BEFORE "{}")',
'date__on': '(ON "{}")',
'subject': '(SUBJECT "{}")',
'uid__range': '(UID {})',
}

FOLDER_LOOKUP = {}

def __init__(self,
connection,
Expand All @@ -28,8 +43,8 @@ def _fetch_email(self, uid):
parser_policy=self.parser_policy)

def _query_uids(self, **kwargs):
query_ = build_search_query(**kwargs)
message, data = self.connection.uid('search', None, query_)
query_ = build_search_query(self.IMAP_ATTRIBUTE_LOOKUP, **kwargs)
_, data = self.connection.uid('search', None, query_)
if data[0] is None:
return []
return data[0].split()
Expand Down
14 changes: 9 additions & 5 deletions imbox/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ def get_mail_addresses(message, header_name):
for index, (address_name, address_email) in enumerate(addresses):
addresses[index] = {'name': decode_mail_header(address_name),
'email': address_email}
logger.debug("{} Mail address in message: <{}> {}".format(header_name.upper(), address_name, address_email))
logger.debug("{} Mail address in message: <{}> {}".format(
header_name.upper(), address_name, address_email))
return addresses


Expand Down Expand Up @@ -111,7 +112,8 @@ def parse_attachment(message_part):
name, value = decode_param(param)

if 'file' in name:
attachment['filename'] = value[1:-1] if value.startswith('"') else value
attachment['filename'] = value[1:-
1] if value.startswith('"') else value

if 'create-date' in name:
attachment['create-date'] = value
Expand Down Expand Up @@ -163,9 +165,11 @@ def parse_email(raw_email, policy=None):
email_parse_kwargs = {}

try:
email_message = email.message_from_string(raw_email, **email_parse_kwargs)
email_message = email.message_from_string(
raw_email, **email_parse_kwargs)
except UnicodeEncodeError:
email_message = email.message_from_string(raw_email.encode('utf-8'), **email_parse_kwargs)
email_message = email.message_from_string(
raw_email.encode('utf-8'), **email_parse_kwargs)
maintype = email_message.get_content_maintype()
parsed_email = {'raw_email': raw_email}

Expand All @@ -187,7 +191,7 @@ def parse_email(raw_email, policy=None):
content = decode_content(part)

is_inline = content_disposition is None \
or content_disposition.startswith("inline")
or content_disposition.startswith("inline")
if content_type == "text/plain" and is_inline:
body['plain'].append(content)
elif content_type == "text/html" and is_inline:
Expand Down
64 changes: 8 additions & 56 deletions imbox/query.py
Original file line number Diff line number Diff line change
@@ -1,65 +1,17 @@
import datetime
import logging
# TODO - Validate query arguments

logger = logging.getLogger(__name__)


def format_date(date):
if isinstance(date, datetime.date):
return date.strftime('%d-%b-%Y')
return date


def build_search_query(**kwargs):

# Parse keyword arguments
unread = kwargs.get('unread', False)
unflagged = kwargs.get('unflagged', False)
flagged = kwargs.get('flagged', False)
sent_from = kwargs.get('sent_from', False)
sent_to = kwargs.get('sent_to', False)
date__gt = kwargs.get('date__gt', False)
date__lt = kwargs.get('date__lt', False)
date__on = kwargs.get('date__on', False)
subject = kwargs.get('subject')
uid__range = kwargs.get('uid__range')

def build_search_query(imap_attribute_lookup, **kwargs):
query = []

if unread:
query.append("(UNSEEN)")

if unflagged:
query.append("(UNFLAGGED)")

if flagged:
query.append("(FLAGGED)")

if sent_from:
query.append('(FROM "%s")' % sent_from)

if sent_to:
query.append('(TO "%s")' % sent_to)

if date__gt:
query.append('(SINCE "%s")' % format_date(date__gt))

if date__lt:
query.append('(BEFORE "%s")' % format_date(date__lt))

if date__on:
query.append('(ON "%s")' % format_date(date__on))

if subject is not None:
query.append('(SUBJECT "%s")' % subject)

if uid__range:
query.append('(UID %s)' % uid__range)
for name, value in kwargs.items():
if value is not None:
if isinstance(value, datetime.date):
value = value.strftime('%d-%b-%Y')
if isinstance(value, str) and '"' in value:
value = value.replace('"', "'")
query.append(imap_attribute_lookup[name].format(value))

if query:
logger.debug("IMAP query: {}".format(" ".join(query)))
return " ".join(query)

logger.debug("IMAP query: {}".format("(ALL)"))
return "(ALL)"
7 changes: 0 additions & 7 deletions imbox/query.pyi

This file was deleted.

12 changes: 11 additions & 1 deletion imbox/vendors/gmail.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
from imbox.messages import Messages
from imbox.vendors.helpers import merge_two_dicts


class GmailMessages(Messages):
authentication_error_message = ('If you\'re not using an app-specific password, grab one here: '
'https://myaccount.google.com/apppasswords')
hostname = 'imap.gmail.com'
name = 'gmail'
folder_lookup = {
FOLDER_LOOKUP = {

'all_mail': '"[Gmail]/All Mail"',
'all': '"[Gmail]/All Mail"',
Expand All @@ -19,11 +20,20 @@ class GmailMessages(Messages):
'spam': '"[Gmail]/Spam"',
'starred': '"[Gmail]/Starred"',
'trash': '"[Gmail]/Trash"',
}

GMAIL_IMAP_ATTRIBUTE_LOOKUP_DIFF = {
'subject': '(X-GM-RAW "subject:\'{}\'")',
'label': '(X-GM-LABELS "{}")',
'raw': '(X-GM-RAW "{}")'
}

def __init__(self,
connection,
parser_policy,
**kwargs):

self.IMAP_ATTRIBUTE_LOOKUP = merge_two_dicts(self.IMAP_ATTRIBUTE_LOOKUP,
self.GMAIL_IMAP_ATTRIBUTE_LOOKUP_DIFF)

super().__init__(connection, parser_policy, **kwargs)
6 changes: 6 additions & 0 deletions imbox/vendors/helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@

def merge_two_dicts(x, y):
"""from https://stackoverflow.com/a/26853961/4386191"""
z = x.copy() # start with x's keys and values
z.update(y) # modifies z with y's keys and values & returns None
return z
51 changes: 40 additions & 11 deletions tests/query_tests.py
Original file line number Diff line number Diff line change
@@ -1,54 +1,83 @@
from datetime import date
import unittest

from imbox.query import build_search_query
from datetime import date
from imbox.messages import Messages
from imbox.vendors.helpers import merge_two_dicts
from imbox.vendors.gmail import GmailMessages

IMAP_ATTRIBUTE_LOOKUP = Messages.IMAP_ATTRIBUTE_LOOKUP
GMAIL_ATTRIBUTE_LOOKUP = merge_two_dicts(IMAP_ATTRIBUTE_LOOKUP,
GmailMessages.GMAIL_IMAP_ATTRIBUTE_LOOKUP_DIFF)


class TestQuery(unittest.TestCase):

def test_all(self):

res = build_search_query()
res = build_search_query(IMAP_ATTRIBUTE_LOOKUP)
self.assertEqual(res, "(ALL)")

def test_subject(self):

res = build_search_query(IMAP_ATTRIBUTE_LOOKUP, subject='hi')
self.assertEqual(res, '(SUBJECT "hi")')

res = build_search_query(GMAIL_ATTRIBUTE_LOOKUP, subject='hi')
self.assertEqual(res, '(X-GM-RAW "subject:\'hi\'")')

def test_unread(self):

res = build_search_query(unread=True)
res = build_search_query(IMAP_ATTRIBUTE_LOOKUP, unread=True)
self.assertEqual(res, "(UNSEEN)")

def test_unflagged(self):

res = build_search_query(unflagged=True)
res = build_search_query(IMAP_ATTRIBUTE_LOOKUP, unflagged=True)
self.assertEqual(res, "(UNFLAGGED)")

def test_flagged(self):

res = build_search_query(flagged=True)
res = build_search_query(IMAP_ATTRIBUTE_LOOKUP, flagged=True)
self.assertEqual(res, "(FLAGGED)")

def test_sent_from(self):

res = build_search_query(sent_from='test@example.com')
res = build_search_query(
IMAP_ATTRIBUTE_LOOKUP, sent_from='test@example.com')
self.assertEqual(res, '(FROM "test@example.com")')

def test_sent_to(self):

res = build_search_query(sent_to='test@example.com')
res = build_search_query(
IMAP_ATTRIBUTE_LOOKUP, sent_to='test@example.com')
self.assertEqual(res, '(TO "test@example.com")')

def test_date__gt(self):

res = build_search_query(date__gt=date(2014, 12, 31))
res = build_search_query(
IMAP_ATTRIBUTE_LOOKUP, date__gt=date(2014, 12, 31))
self.assertEqual(res, '(SINCE "31-Dec-2014")')

def test_date__lt(self):

res = build_search_query(date__lt=date(2014, 1, 1))
res = build_search_query(
IMAP_ATTRIBUTE_LOOKUP, date__lt=date(2014, 1, 1))
self.assertEqual(res, '(BEFORE "01-Jan-2014")')

def test_date__on(self):
res = build_search_query(date__on=date(2014, 1, 1))
res = build_search_query(
IMAP_ATTRIBUTE_LOOKUP, date__on=date(2014, 1, 1))
self.assertEqual(res, '(ON "01-Jan-2014")')

def test_uid__range(self):
res = build_search_query(uid__range='1000:*')
res = build_search_query(IMAP_ATTRIBUTE_LOOKUP, uid__range='1000:*')
self.assertEqual(res, '(UID 1000:*)')

def test_gmail_raw(self):
res = build_search_query(GMAIL_ATTRIBUTE_LOOKUP, raw='has:attachment subject:"hey"')
self.assertEqual(res, '(X-GM-RAW "has:attachment subject:\'hey\'")')

def test_gmail_label(self):
res = build_search_query(GMAIL_ATTRIBUTE_LOOKUP, label='finance')
self.assertEqual(res, '(X-GM-LABELS "finance")')

0 comments on commit 8d7d133

Please sign in to comment.