From c53d9152a2283aa4fd1122a5ce849270a51593dc Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 10 Mar 2014 10:21:39 +0100 Subject: [PATCH 01/16] Allow for complex query, for instance on UID --- gmail/mailbox.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gmail/mailbox.py b/gmail/mailbox.py index 0368034..4299977 100644 --- a/gmail/mailbox.py +++ b/gmail/mailbox.py @@ -55,11 +55,12 @@ def mail(self, prefetch=False, **kwargs): kwargs.get('attachment') and search.extend(['HAS', 'attachment']) kwargs.get('query') and search.extend([kwargs.get('query')]) + kwargs.get('custom_query') and search.extend(kwargs.get('custom_query')) emails = [] # print search response, data = self.gmail.imap.uid('SEARCH', *search) - if response == 'OK': + if response == 'OK': uids = filter(None, data[0].split(' ')) # filter out empty strings for uid in uids: @@ -79,8 +80,8 @@ def mail(self, prefetch=False, **kwargs): def threads(self, prefetch=False, **kwargs): emails = [] response, data = self.gmail.imap.uid('SEARCH', 'ALL') - if response == 'OK': - uids = data[0].split(' ') + if response == 'OK': + uids = data[0].split(' ') for uid in uids: From fa6b381f801ef0ea97929b568dfc0626ee8cf3a4 Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 10 Mar 2014 10:23:05 +0100 Subject: [PATCH 02/16] Message and body encoding --- gmail/message.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 9813ce4..41d6f97 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -7,8 +7,6 @@ from imaplib import ParseFlags class Message(): - - def __init__(self, mailbox, uid): self.uid = uid self.mailbox = mailbox @@ -34,10 +32,8 @@ def __init__(self, mailbox, uid): self.thread_id = None self.thread = [] self.message_id = None - - self.attachments = None - + self.attachments = None def is_read(self): return ('\\Seen' in self.flags) @@ -82,7 +78,6 @@ def remove_label(self, label): self.gmail.imap.uid('STORE', self.uid, '-X-GM-LABELS', full_label) if full_label in self.labels: self.labels.remove(full_label) - def is_deleted(self): return ('\\Deleted' in self.flags) @@ -106,8 +101,6 @@ def move_to(self, name): if name not in ['[Gmail]/Bin', '[Gmail]/Trash']: self.delete() - - def archive(self): self.move_to('[Gmail]/All Mail') @@ -150,8 +143,12 @@ def parse(self, raw_message): for content in self.message.walk(): if content.get_content_type() == "text/plain": self.body = content.get_payload(decode=True) + if content.get_content_charset(): + self.body = unicode(self.body, content.get_content_charset(), 'ignore').encode('utf8', 'replace') elif content.get_content_type() == "text/html": self.html = content.get_payload(decode=True) + if content.get_content_charset(): + self.html = unicode(self.html, content.get_content_charset(), 'ignore').encode('utf8', 'replace') elif self.message.get_content_maintype() == "text": self.body = self.message.get_payload() @@ -166,13 +163,13 @@ def parse(self, raw_message): if re.search(r'X-GM-MSGID (\d+)', raw_headers): self.message_id = re.search(r'X-GM-MSGID (\d+)', raw_headers).groups(1)[0] - + # Parse attachments into attachment objects array for this message self.attachments = [ Attachment(attachment) for attachment in self.message._payload if not isinstance(attachment, basestring) and attachment.get('Content-Disposition') is not None ] - + def fetch(self): if not self.message: From 297f50e2f048f2d5ee23ab32e0de64caa4838548 Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 10 Mar 2014 12:56:46 +0100 Subject: [PATCH 03/16] Store attrs on mailbox --- gmail/gmail.py | 4 +++- gmail/mailbox.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/gmail/gmail.py b/gmail/gmail.py index 7fb09e1..935876a 100644 --- a/gmail/gmail.py +++ b/gmail/gmail.py @@ -54,8 +54,10 @@ def fetch_mailboxes(self): if response == 'OK': for mailbox in mailbox_list: mailbox_name = mailbox.split('"/"')[-1].replace('"', '').strip() + mailbox_attrs = mailbox.split('"/"')[0] mailbox = Mailbox(self) mailbox.external_name = mailbox_name + mailbox.attrs = mailbox_attrs self.mailboxes[mailbox_name] = mailbox def use_mailbox(self, mailbox): @@ -140,7 +142,7 @@ def find(self, mailbox_name="[Gmail]/All Mail", **kwargs): box = self.mailbox(mailbox_name) return box.mail(**kwargs) - + def copy(self, uid, to_mailbox, from_mailbox=None): if from_mailbox: self.use_mailbox(from_mailbox) diff --git a/gmail/mailbox.py b/gmail/mailbox.py index 4299977..d5ceb9a 100644 --- a/gmail/mailbox.py +++ b/gmail/mailbox.py @@ -9,6 +9,7 @@ def __init__(self, gmail, name="INBOX"): self.gmail = gmail self.date_format = "%d-%b-%Y" self.messages = {} + self.attrs = "" @property def external_name(self): From ca143b2322b6ab396186b27428ac00289c68faba Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 10 Mar 2014 14:01:41 +0100 Subject: [PATCH 04/16] Proper encoding for non multipart messages --- gmail/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gmail/message.py b/gmail/message.py index 41d6f97..0a13f3e 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -150,7 +150,9 @@ def parse(self, raw_message): if content.get_content_charset(): self.html = unicode(self.html, content.get_content_charset(), 'ignore').encode('utf8', 'replace') elif self.message.get_content_maintype() == "text": - self.body = self.message.get_payload() + self.body = self.message.get_payload(decode=True) + if self.message.get_content_charset(): + self.body = unicode(self.body, self.message.get_content_charset(), 'ignore').encode('utf8', 'replace') self.sent_at = datetime.datetime.fromtimestamp(time.mktime(email.utils.parsedate_tz(self.message['date'])[:9])) From bb905cb30756c0d1b07e5a2e3e9a9a8e13aa1d4a Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 10 Mar 2014 15:26:15 +0100 Subject: [PATCH 05/16] Improved parser --- __init__.py | 0 gmail/message.py | 33 ++++++++++++++++----------------- 2 files changed, 16 insertions(+), 17 deletions(-) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gmail/message.py b/gmail/message.py index 0a13f3e..131e155 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -3,6 +3,7 @@ import re import time import os + from email.header import decode_header, make_header from imaplib import ParseFlags @@ -121,8 +122,8 @@ def parse_labels(self, headers): else: return list() - def parse_subject(self, encoded_subject): - dh = decode_header(encoded_subject) + def parse_header(self, encoded_header): + dh = decode_header(encoded_header) default_charset = 'ASCII' return ''.join([ unicode(t[0], t[1] or default_charset) for t in dh ]) @@ -131,28 +132,27 @@ def parse(self, raw_message): raw_email = raw_message[1] self.message = email.message_from_string(raw_email) - self.headers = self.parse_headers(self.message) - self.to = self.message['to'] - self.fr = self.message['from'] - self.delivered_to = self.message['delivered_to'] + def to_unicode(value, charset): + r = value + if charset: + r = unicode(r, charset, 'ignore').encode('utf8', 'replace') + return r - self.subject = self.parse_subject(self.message['subject']) + self.headers = self.parse_headers(self.message) + + self.to = self.parse_header(self.message['to']) + self.fr = self.parse_header(self.message['from']) + self.subject = self.parse_header(self.message['subject']) if self.message.get_content_maintype() == "multipart": for content in self.message.walk(): if content.get_content_type() == "text/plain": - self.body = content.get_payload(decode=True) - if content.get_content_charset(): - self.body = unicode(self.body, content.get_content_charset(), 'ignore').encode('utf8', 'replace') + self.body = to_unicode(content.get_payload(decode=True), content.get_content_charset()) elif content.get_content_type() == "text/html": - self.html = content.get_payload(decode=True) - if content.get_content_charset(): - self.html = unicode(self.html, content.get_content_charset(), 'ignore').encode('utf8', 'replace') + self.html = to_unicode(content.get_payload(decode=True), content.get_content_charset()) elif self.message.get_content_maintype() == "text": - self.body = self.message.get_payload(decode=True) - if self.message.get_content_charset(): - self.body = unicode(self.body, self.message.get_content_charset(), 'ignore').encode('utf8', 'replace') + self.body = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) self.sent_at = datetime.datetime.fromtimestamp(time.mktime(email.utils.parsedate_tz(self.message['date'])[:9])) @@ -165,7 +165,6 @@ def parse(self, raw_message): if re.search(r'X-GM-MSGID (\d+)', raw_headers): self.message_id = re.search(r'X-GM-MSGID (\d+)', raw_headers).groups(1)[0] - # Parse attachments into attachment objects array for this message self.attachments = [ Attachment(attachment) for attachment in self.message._payload From bad244d28d7de1afe4ce77593db9e32a73b3f7b9 Mon Sep 17 00:00:00 2001 From: Neamar Date: Thu, 3 Apr 2014 20:30:19 +0200 Subject: [PATCH 06/16] Improved unicode --- gmail/message.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 131e155..444e002 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -122,10 +122,21 @@ def parse_labels(self, headers): else: return list() + def try_parse(self, header, encoding): + if encoding is None: + encoding = 'ASCII' + + try: + return unicode(header, encoding) + except UnicodeDecodeError: + try: + return unicode(header, 'ISO-8859-1') + except UnicodeDecodeError: + return unicode(header, 'UTF-8') + def parse_header(self, encoded_header): dh = decode_header(encoded_header) - default_charset = 'ASCII' - return ''.join([ unicode(t[0], t[1] or default_charset) for t in dh ]) + return ''.join([self.try_parse(t[0], t[1]) for t in dh]) def parse(self, raw_message): raw_headers = raw_message[0] @@ -134,9 +145,8 @@ def parse(self, raw_message): self.message = email.message_from_string(raw_email) def to_unicode(value, charset): - r = value - if charset: - r = unicode(r, charset, 'ignore').encode('utf8', 'replace') + r = self.try_parse(value, charset) + return r self.headers = self.parse_headers(self.message) From b85275630e7a09911df37d3bcd8dd4e2ef6fa68b Mon Sep 17 00:00:00 2001 From: Neamar Date: Tue, 8 Apr 2014 15:07:04 +0200 Subject: [PATCH 07/16] Skip empty attachments --- gmail/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gmail/message.py b/gmail/message.py index 444e002..0503599 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -178,7 +178,7 @@ def to_unicode(value, charset): # Parse attachments into attachment objects array for this message self.attachments = [ Attachment(attachment) for attachment in self.message._payload - if not isinstance(attachment, basestring) and attachment.get('Content-Disposition') is not None + if not isinstance(attachment, basestring) and attachment.get('Content-Disposition') is not None and attachment.get_filename() is not None ] From bacebed820d485d3ab0f154768bc5264ec7051d5 Mon Sep 17 00:00:00 2001 From: Neamar Date: Fri, 11 Apr 2014 11:49:36 +0200 Subject: [PATCH 08/16] PEP8 --- gmail/__init__.py | 5 ++--- gmail/exceptions.py | 3 +++ gmail/gmail.py | 12 ++---------- gmail/mailbox.py | 19 ++++++++++--------- gmail/message.py | 34 +++++++++++++++++++++------------- gmail/utf.py | 10 +++++++--- gmail/utils.py | 6 +++--- 7 files changed, 48 insertions(+), 41 deletions(-) diff --git a/gmail/__init__.py b/gmail/__init__.py index f90b763..e3c290b 100644 --- a/gmail/__init__.py +++ b/gmail/__init__.py @@ -14,8 +14,7 @@ __copyright__ = 'Copyright 2013 Charlie Guo' from .gmail import Gmail -from .mailbox import Mailbox -from .message import Message +from .mailbox import Mailbox +from .message import Message from .exceptions import GmailException, ConnectionError, AuthenticationError from .utils import login, authenticate - diff --git a/gmail/exceptions.py b/gmail/exceptions.py index 3306d87..cc89155 100644 --- a/gmail/exceptions.py +++ b/gmail/exceptions.py @@ -13,11 +13,14 @@ class GmailException(RuntimeError): """There was an ambiguous exception that occurred while handling your request.""" + class ConnectionError(GmailException): """A Connection error occurred.""" + class AuthenticationError(GmailException): """Gmail Authentication failed.""" + class Timeout(GmailException): """The request timed out.""" diff --git a/gmail/gmail.py b/gmail/gmail.py index 935876a..ea63cb8 100644 --- a/gmail/gmail.py +++ b/gmail/gmail.py @@ -5,6 +5,7 @@ from utf import encode as encode_utf7, decode as decode_utf7 from exceptions import * + class Gmail(): # GMail IMAP defaults GMAIL_IMAP_HOST = 'imap.gmail.com' @@ -26,10 +27,8 @@ def __init__(self): self.mailboxes = {} self.current_mailbox = None - # self.connect() - def connect(self, raise_errors=True): # try: # self.imap = imaplib.IMAP4_SSL(self.GMAIL_IMAP_HOST, self.GMAIL_IMAP_PORT) @@ -48,7 +47,6 @@ def connect(self, raise_errors=True): return self.imap - def fetch_mailboxes(self): response, mailbox_list = self.imap.list() if response == 'OK': @@ -90,8 +88,6 @@ def delete_mailbox(self, mailbox_name): self.imap.delete(mailbox_name) del self.mailboxes[mailbox_name] - - def login(self, username, password): self.username = username self.password = password @@ -107,7 +103,6 @@ def login(self, username, password): except imaplib.IMAP4.error: raise AuthenticationError - # smtp_login(username, password) return self.logged_in @@ -134,7 +129,6 @@ def logout(self): self.imap.logout() self.logged_in = False - def label(self, label_name): return self.mailbox(label_name) @@ -142,14 +136,13 @@ def find(self, mailbox_name="[Gmail]/All Mail", **kwargs): box = self.mailbox(mailbox_name) return box.mail(**kwargs) - def copy(self, uid, to_mailbox, from_mailbox=None): if from_mailbox: self.use_mailbox(from_mailbox) self.imap.uid('COPY', uid, to_mailbox) def fetch_multiple_messages(self, messages): - fetch_str = ','.join(messages.keys()) + fetch_str = ','.join(messages.keys()) response, results = self.imap.uid('FETCH', fetch_str, '(BODY.PEEK[] FLAGS X-GM-THRID X-GM-MSGID X-GM-LABELS)') for index in xrange(len(results) - 1): raw_message = results[index] @@ -159,7 +152,6 @@ def fetch_multiple_messages(self, messages): return messages - def labels(self, require_unicode=False): keys = self.mailboxes.keys() if require_unicode: diff --git a/gmail/mailbox.py b/gmail/mailbox.py index d5ceb9a..c455d2e 100644 --- a/gmail/mailbox.py +++ b/gmail/mailbox.py @@ -1,6 +1,8 @@ -from message import Message +import re from utf import encode as encode_utf7, decode as decode_utf7 +from message import Message + class Mailbox(): @@ -26,21 +28,21 @@ def external_name(self, value): def mail(self, prefetch=False, **kwargs): search = ['ALL'] - kwargs.get('read') and search.append('SEEN') + kwargs.get('read') and search.append('SEEN') kwargs.get('unread') and search.append('UNSEEN') - kwargs.get('starred') and search.append('FLAGGED') + kwargs.get('starred') and search.append('FLAGGED') kwargs.get('unstarred') and search.append('UNFLAGGED') - kwargs.get('deleted') and search.append('DELETED') + kwargs.get('deleted') and search.append('DELETED') kwargs.get('undeleted') and search.append('UNDELETED') - kwargs.get('draft') and search.append('DRAFT') + kwargs.get('draft') and search.append('DRAFT') kwargs.get('undraft') and search.append('UNDRAFT') kwargs.get('before') and search.extend(['BEFORE', kwargs.get('before').strftime(self.date_format)]) - kwargs.get('after') and search.extend(['SINCE', kwargs.get('after').strftime(self.date_format)]) - kwargs.get('on') and search.extend(['ON', kwargs.get('on').strftime(self.date_format)]) + kwargs.get('after') and search.extend(['SINCE', kwargs.get('after').strftime(self.date_format)]) + kwargs.get('on') and search.extend(['ON', kwargs.get('on').strftime(self.date_format)]) kwargs.get('header') and search.extend(['HEADER', kwargs.get('header')[0], kwargs.get('header')[1]]) @@ -62,7 +64,7 @@ def mail(self, prefetch=False, **kwargs): # print search response, data = self.gmail.imap.uid('SEARCH', *search) if response == 'OK': - uids = filter(None, data[0].split(' ')) # filter out empty strings + uids = filter(None, data[0].split(' ')) # filter out empty strings for uid in uids: if not self.messages.get(uid): @@ -84,7 +86,6 @@ def threads(self, prefetch=False, **kwargs): if response == 'OK': uids = data[0].split(' ') - for uid in uids: if not self.messages.get(uid): self.messages[uid] = Message(self, uid) diff --git a/gmail/message.py b/gmail/message.py index 0503599..f0c5310 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -4,9 +4,10 @@ import time import os -from email.header import decode_header, make_header +from email.header import decode_header from imaplib import ParseFlags + class Message(): def __init__(self, mailbox, uid): self.uid = uid @@ -42,12 +43,14 @@ def is_read(self): def read(self): flag = '\\Seen' self.gmail.imap.uid('STORE', self.uid, '+FLAGS', flag) - if flag not in self.flags: self.flags.append(flag) + if flag not in self.flags: + self.flags.append(flag) def unread(self): flag = '\\Seen' self.gmail.imap.uid('STORE', self.uid, '-FLAGS', flag) - if flag in self.flags: self.flags.remove(flag) + if flag in self.flags: + self.flags.remove(flag) def is_starred(self): return ('\\Flagged' in self.flags) @@ -55,12 +58,14 @@ def is_starred(self): def star(self): flag = '\\Flagged' self.gmail.imap.uid('STORE', self.uid, '+FLAGS', flag) - if flag not in self.flags: self.flags.append(flag) + if flag not in self.flags: + self.flags.append(flag) def unstar(self): flag = '\\Flagged' self.gmail.imap.uid('STORE', self.uid, '-FLAGS', flag) - if flag in self.flags: self.flags.remove(flag) + if flag in self.flags: + self.flags.remove(flag) def is_draft(self): return ('\\Draft' in self.flags) @@ -72,12 +77,14 @@ def has_label(self, label): def add_label(self, label): full_label = '%s' % label self.gmail.imap.uid('STORE', self.uid, '+X-GM-LABELS', full_label) - if full_label not in self.labels: self.labels.append(full_label) + if full_label not in self.labels: + self.labels.append(full_label) def remove_label(self, label): full_label = '%s' % label self.gmail.imap.uid('STORE', self.uid, '-X-GM-LABELS', full_label) - if full_label in self.labels: self.labels.remove(full_label) + if full_label in self.labels: + self.labels.remove(full_label) def is_deleted(self): return ('\\Deleted' in self.flags) @@ -85,7 +92,8 @@ def is_deleted(self): def delete(self): flag = '\\Deleted' self.gmail.imap.uid('STORE', self.uid, '+FLAGS', flag) - if flag not in self.flags: self.flags.append(flag) + if flag not in self.flags: + self.flags.append(flag) trash = '[Gmail]/Trash' if '[Gmail]/Trash' in self.gmail.labels() else '[Gmail]/Bin' if self.mailbox.name not in ['[Gmail]/Bin', '[Gmail]/Trash']: @@ -96,7 +104,6 @@ def delete(self): # self.gmail.imap.uid('STORE', self.uid, '-FLAGS', flag) # if flag in self.flags: self.flags.remove(flag) - def move_to(self, name): self.gmail.copy(self.uid, name, self.mailbox.name) if name not in ['[Gmail]/Bin', '[Gmail]/Trash']: @@ -181,7 +188,6 @@ def to_unicode(value, charset): if not isinstance(attachment, basestring) and attachment.get('Content-Disposition') is not None and attachment.get_filename() is not None ] - def fetch(self): if not self.message: response, results = self.gmail.imap.uid('FETCH', self.uid, '(BODY.PEEK[] FLAGS X-GM-THRID X-GM-MSGID X-GM-LABELS)') @@ -201,7 +207,8 @@ def fetch_thread(self): received_messages = {} uids = results[0].split(' ') if response == 'OK': - for uid in uids: received_messages[uid] = Message(original_mailbox, uid) + for uid in uids: + received_messages[uid] = Message(original_mailbox, uid) self.gmail.fetch_multiple_messages(received_messages) self.mailbox.messages.update(received_messages) @@ -211,7 +218,8 @@ def fetch_thread(self): sent_messages = {} uids = results[0].split(' ') if response == 'OK': - for uid in uids: sent_messages[uid] = Message(self.gmail.mailboxes['[Gmail]/Sent Mail'], uid) + for uid in uids: + sent_messages[uid] = Message(self.gmail.mailboxes['[Gmail]/Sent Mail'], uid) self.gmail.fetch_multiple_messages(sent_messages) self.gmail.mailboxes['[Gmail]/Sent Mail'].messages.update(sent_messages) @@ -228,7 +236,7 @@ def __init__(self, attachment): # Raw file data self.payload = attachment.get_payload(decode=True) # Filesize in kilobytes - self.size = int(round(len(self.payload)/1000.0)) + self.size = int(round(len(self.payload) / 1000.0)) def save(self, path=None): if path is None: diff --git a/gmail/utf.py b/gmail/utf.py index c953e73..5a6864b 100644 --- a/gmail/utf.py +++ b/gmail/utf.py @@ -10,10 +10,10 @@ # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: -# +# # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. -# +# # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND @@ -27,6 +27,7 @@ PRINTABLE = set(range(0x20, 0x26)) | set(range(0x27, 0x7f)) + def encode(s): """Encode a folder name using IMAP modified UTF-7 encoding. @@ -57,6 +58,7 @@ def extend_result_if_chars_buffered(): return ''.join(r) + def decode(s): """Decode a folder name from IMAP modified UTF-7 encoding to unicode. @@ -88,12 +90,14 @@ def decode(s): return ''.join(r) + def modified_utf7(s): # encode to utf-7: '\xff' => b'+AP8-', decode from latin-1 => '+AP8-' s_utf7 = s.encode('utf-7').decode('latin-1') return s_utf7[1:-1].replace('/', ',') + def modified_deutf7(s): s_utf7 = '+' + s.replace(',', '/') + '-' # encode to latin-1: '+AP8-' => b'+AP8-', decode from utf-7 => '\xff' - return s_utf7.encode('latin-1').decode('utf-7') \ No newline at end of file + return s_utf7.encode('latin-1').decode('utf-7') diff --git a/gmail/utils.py b/gmail/utils.py index 5b37e2b..b87bdbc 100644 --- a/gmail/utils.py +++ b/gmail/utils.py @@ -1,13 +1,13 @@ +from .gmail import Gmail -from .gmail import Gmail - def login(username, password): gmail = Gmail() gmail.login(username, password) return gmail + def authenticate(username, access_token): gmail = Gmail() gmail.authenticate(username, access_token) - return gmail \ No newline at end of file + return gmail From 615555baf894484f219a537326c0c97406541a34 Mon Sep 17 00:00:00 2001 From: Neamar Date: Fri, 11 Apr 2014 12:11:51 +0200 Subject: [PATCH 09/16] Handle EML attachments --- gmail/message.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index f0c5310..2acf950 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -230,13 +230,17 @@ def fetch_thread(self): class Attachment: - def __init__(self, attachment): self.name = attachment.get_filename() # Raw file data - self.payload = attachment.get_payload(decode=True) - # Filesize in kilobytes - self.size = int(round(len(self.payload) / 1000.0)) + if isinstance(attachment.get_payload(), basestring): + self.payload = attachment.get_payload(decode=True) + # Filesize in kilobytes + self.size = int(round(len(self.payload) / 1000.0)) + else: + # Special case. Seems to occurs only for EML attachments. + self.payload = None + self.size = None def save(self, path=None): if path is None: From 8da972ac2d313435226f2745d9f719ab73cbdb36 Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 28 Apr 2014 14:02:30 +0200 Subject: [PATCH 10/16] Catch Lookup errors --- gmail/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gmail/message.py b/gmail/message.py index 2acf950..334fbe6 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -135,7 +135,7 @@ def try_parse(self, header, encoding): try: return unicode(header, encoding) - except UnicodeDecodeError: + except (UnicodeDecodeError, LookupError): try: return unicode(header, 'ISO-8859-1') except UnicodeDecodeError: From ff8acc26230e543109527f44057f24d9ea329a17 Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 28 Apr 2014 14:33:34 +0200 Subject: [PATCH 11/16] Parse attachment filename --- gmail/message.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 334fbe6..1df8a8e 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -8,6 +8,24 @@ from imaplib import ParseFlags +def try_parse(header, encoding="ASCII"): + """ + Try to decode specified header using specified encoding. + On failure to do so, use ISO-8859-1, and then UTF-8. + Header and encoding are most often return from decode_header. + """ + if encoding is None: + encoding = 'ASCII' + + try: + return unicode(header, encoding) + except (UnicodeDecodeError, LookupError): + try: + return unicode(header, 'ISO-8859-1') + except UnicodeDecodeError: + return unicode(header, 'UTF-8') + + class Message(): def __init__(self, mailbox, uid): self.uid = uid @@ -129,21 +147,9 @@ def parse_labels(self, headers): else: return list() - def try_parse(self, header, encoding): - if encoding is None: - encoding = 'ASCII' - - try: - return unicode(header, encoding) - except (UnicodeDecodeError, LookupError): - try: - return unicode(header, 'ISO-8859-1') - except UnicodeDecodeError: - return unicode(header, 'UTF-8') - def parse_header(self, encoded_header): dh = decode_header(encoded_header) - return ''.join([self.try_parse(t[0], t[1]) for t in dh]) + return ''.join([try_parse(t[0], t[1]) for t in dh]) def parse(self, raw_message): raw_headers = raw_message[0] @@ -152,7 +158,7 @@ def parse(self, raw_message): self.message = email.message_from_string(raw_email) def to_unicode(value, charset): - r = self.try_parse(value, charset) + r = try_parse(value, charset) return r @@ -231,7 +237,9 @@ def fetch_thread(self): class Attachment: def __init__(self, attachment): - self.name = attachment.get_filename() + dh = decode_header(attachment.get_filename()) + self.name = ''.join([try_parse(t[0], t[1]) for t in dh]) + # Raw file data if isinstance(attachment.get_payload(), basestring): self.payload = attachment.get_payload(decode=True) From db3d962984f0e97f8b9781e184666618f51c03b1 Mon Sep 17 00:00:00 2001 From: Neamar Date: Mon, 28 Apr 2014 15:02:44 +0200 Subject: [PATCH 12/16] Missing date header --- gmail/message.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gmail/message.py b/gmail/message.py index 1df8a8e..07c2570 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -177,7 +177,10 @@ def to_unicode(value, charset): elif self.message.get_content_maintype() == "text": self.body = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) - self.sent_at = datetime.datetime.fromtimestamp(time.mktime(email.utils.parsedate_tz(self.message['date'])[:9])) + try: + self.sent_at = datetime.datetime.fromtimestamp(time.mktime(email.utils.parsedate_tz(self.message['date'])[:9])) + except: + self.sent_at = datetime.datetime.now() self.flags = self.parse_flags(raw_headers) From 35b755b5b471fe41532db4e4196958a161acaaab Mon Sep 17 00:00:00 2001 From: Neamar Date: Wed, 30 Apr 2014 13:38:29 +0200 Subject: [PATCH 13/16] Bad-encoded attachment name --- gmail/message.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 07c2570..3aa9fae 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -240,8 +240,11 @@ def fetch_thread(self): class Attachment: def __init__(self, attachment): - dh = decode_header(attachment.get_filename()) - self.name = ''.join([try_parse(t[0], t[1]) for t in dh]) + try: + dh = decode_header(attachment.get_filename()) + self.name = ''.join([try_parse(t[0], t[1]) for t in dh]) + except UnicodeEncodeError: + self.name = attachment.get_filename() # Raw file data if isinstance(attachment.get_payload(), basestring): From 6b76f7b79a7357f65a5fe1059750a0294c6160a0 Mon Sep 17 00:00:00 2001 From: Neamar Date: Wed, 7 May 2014 15:05:08 +0200 Subject: [PATCH 14/16] Imap is a fucking fuckanus --- gmail/message.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 3aa9fae..98b676c 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -4,15 +4,15 @@ import time import os -from email.header import decode_header +from email.header import decode_header, HeaderParseError from imaplib import ParseFlags def try_parse(header, encoding="ASCII"): """ - Try to decode specified header using specified encoding. + Try to parse specified header using specified encoding. On failure to do so, use ISO-8859-1, and then UTF-8. - Header and encoding are most often return from decode_header. + Header and encoding are most often return from decode_header (but use try_decode instead of decode_header, see below). """ if encoding is None: encoding = 'ASCII' @@ -26,6 +26,21 @@ def try_parse(header, encoding="ASCII"): return unicode(header, 'UTF-8') +def try_decode(header): + """ + Try to decode specified header, + We need to wrap this in a try / except for the very rare case of FUCKED UP clients + using FUCKING non-standard base63 encoding. Wow. Such smart. Fucking faglords. + Please phpmailer.codeworxtech.com fix this shit or i'll fucking kill your family and set your house on fire. + See http://bugs.python.org/issue12489 + """ + + try: + return decode_header(header) + except HeaderParseError: + return [[header, None]] + + class Message(): def __init__(self, mailbox, uid): self.uid = uid @@ -148,7 +163,7 @@ def parse_labels(self, headers): return list() def parse_header(self, encoded_header): - dh = decode_header(encoded_header) + dh = try_decode(encoded_header) return ''.join([try_parse(t[0], t[1]) for t in dh]) def parse(self, raw_message): @@ -241,7 +256,7 @@ def fetch_thread(self): class Attachment: def __init__(self, attachment): try: - dh = decode_header(attachment.get_filename()) + dh = try_decode(attachment.get_filename()) self.name = ''.join([try_parse(t[0], t[1]) for t in dh]) except UnicodeEncodeError: self.name = attachment.get_filename() From 17132bb952044fdd701244a7ec95910e99da344d Mon Sep 17 00:00:00 2001 From: Neamar Date: Wed, 21 May 2014 19:08:16 +0200 Subject: [PATCH 15/16] Handle mail without text/plain multipart --- gmail/message.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 98b676c..18b0266 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -190,8 +190,10 @@ def to_unicode(value, charset): elif content.get_content_type() == "text/html": self.html = to_unicode(content.get_payload(decode=True), content.get_content_charset()) elif self.message.get_content_maintype() == "text": - self.body = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) - + if self.message.get_content_type() == "text/plain": + self.body = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) + elif self.message.get_content_type() == "text/html": + self.html = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) try: self.sent_at = datetime.datetime.fromtimestamp(time.mktime(email.utils.parsedate_tz(self.message['date'])[:9])) except: From 94c997216c2c38b0a74949f4403acbf8acc54375 Mon Sep 17 00:00:00 2001 From: Neamar Date: Sun, 1 Jun 2014 11:50:44 +0200 Subject: [PATCH 16/16] Removed useless to_unicode nested function --- gmail/message.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/gmail/message.py b/gmail/message.py index 18b0266..d571ab5 100644 --- a/gmail/message.py +++ b/gmail/message.py @@ -172,11 +172,6 @@ def parse(self, raw_message): self.message = email.message_from_string(raw_email) - def to_unicode(value, charset): - r = try_parse(value, charset) - - return r - self.headers = self.parse_headers(self.message) self.to = self.parse_header(self.message['to']) @@ -186,14 +181,14 @@ def to_unicode(value, charset): if self.message.get_content_maintype() == "multipart": for content in self.message.walk(): if content.get_content_type() == "text/plain": - self.body = to_unicode(content.get_payload(decode=True), content.get_content_charset()) + self.body = try_parse(content.get_payload(decode=True), content.get_content_charset()) elif content.get_content_type() == "text/html": - self.html = to_unicode(content.get_payload(decode=True), content.get_content_charset()) + self.html = try_parse(content.get_payload(decode=True), content.get_content_charset()) elif self.message.get_content_maintype() == "text": if self.message.get_content_type() == "text/plain": - self.body = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) + self.body = try_parse(self.message.get_payload(decode=True), self.message.get_content_charset()) elif self.message.get_content_type() == "text/html": - self.html = to_unicode(self.message.get_payload(decode=True), self.message.get_content_charset()) + self.html = try_parse(self.message.get_payload(decode=True), self.message.get_content_charset()) try: self.sent_at = datetime.datetime.fromtimestamp(time.mktime(email.utils.parsedate_tz(self.message['date'])[:9])) except: