Using pyzmail36 to parse raw mail data

5 files changed, 191 insertions, 167 deletions

diff --git a/Pipfile b/Pipfile
index 36dd8a2..076dc6b 100644
--- a/Pipfile
+++ b/Pipfile
@@ -6,8 +6,10 @@ name = "pypi"
 [packages]
 pytz = "*"
 python-telegram-bot = "*"
+pyzmail36 = "*"
 
 [dev-packages]
+pylint = "*"
 
 [requires]
 python_version = "3.6"
diff --git a/Pipfile.lock b/Pipfile.lock
index 49e9354..1862db6 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "f7da4ab6099df3aff82e024a8a850f8e6606a3ee0ecdbbc55e20e22d40864c5e"
+            "sha256": "65ffd10327170095842dff53978ba96e55245c9e61073a9fca7cc117a9bdf85b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -44,7 +44,100 @@
             ],
             "index": "pypi",
             "version": "==2018.4"
+        },
+        "pyzmail36": {
+            "hashes": [
+                "sha256:4ac22b663a2525422b15de7f1ca5e60983e1adb647debd14ae955963d4d6b098"
+            ],
+            "index": "pypi",
+            "version": "==1.0.3"
         }
     },
-    "develop": {}
+    "develop": {
+        "astroid": {
+            "hashes": [
+                "sha256:0ef2bf9f07c3150929b25e8e61b5198c27b0dca195e156f0e4d5bdd89185ca1a",
+                "sha256:fc9b582dba0366e63540982c3944a9230cbc6f303641c51483fa547dcc22393a"
+            ],
+            "version": "==1.6.5"
+        },
+        "colorama": {
+            "hashes": [
+                "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda",
+                "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==0.3.9"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af",
+                "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8",
+                "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497"
+            ],
+            "version": "==4.3.4"
+        },
+        "lazy-object-proxy": {
+            "hashes": [
+                "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33",
+                "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39",
+                "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019",
+                "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088",
+                "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b",
+                "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e",
+                "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6",
+                "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b",
+                "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5",
+                "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff",
+                "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd",
+                "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7",
+                "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff",
+                "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d",
+                "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2",
+                "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35",
+                "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4",
+                "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514",
+                "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252",
+                "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109",
+                "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f",
+                "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c",
+                "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92",
+                "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577",
+                "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d",
+                "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d",
+                "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f",
+                "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a",
+                "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b"
+            ],
+            "version": "==1.3.1"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
+                "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+            ],
+            "version": "==0.6.1"
+        },
+        "pylint": {
+            "hashes": [
+                "sha256:a48070545c12430cfc4e865bf62f5ad367784765681b3db442d8230f0960aa3c",
+                "sha256:fff220bcb996b4f7e2b0f6812fd81507b72ca4d8c4d05daf2655c333800cb9b3"
+            ],
+            "index": "pypi",
+            "version": "==1.9.2"
+        },
+        "six": {
+            "hashes": [
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+            ],
+            "version": "==1.11.0"
+        },
+        "wrapt": {
+            "hashes": [
+                "sha256:d4d560d479f2c21e1b5443bbd15fe7ec4b37fe7e53d335d3b9b0a7b1226fe3c6"
+            ],
+            "version": "==1.10.11"
+        }
+    }
 }
diff --git a/bot.py b/bot.py
index bda0dfd..85c32d2 100644
--- a/bot.py
+++ b/bot.py
@@ -1,9 +1,8 @@
 import logging
 import os
 
-from utils.mail import EmailClient
+from utils.client import EmailClient
 from telegram import ParseMode
-from telegram.constants import MAX_MESSAGE_LENGTH
 from telegram.ext import (Updater, CommandHandler)
 
 
@@ -49,22 +48,9 @@ def get_email(bot, update, args):
     index = args[0]
     with EmailClient(email_addr, email_passwd) as client:
         mail = client.get_mail_by_index(index)
-        subject = "*Subject*: %s\n" % mail.subject
-        sender = "*From*: %s - %s\n" % (mail.from_nickname, mail.from_account)
-        date = "*Date*: %s\n" % mail.receivedtime
         bot.send_message(update.message.chat_id,
                          parse_mode=ParseMode.MARKDOWN,
-                         text=subject+sender+date)
-        if len(mail.text_content) > MAX_MESSAGE_LENGTH:
-            text = mail.text_content[0:4096]
-            bot.send_message(update.message.chat_id,
-                             text=text)
-            mail.text_content = mail.text_content.lstrip(text)
-        if mail.text_content:
-            bot.send_message(update.message.chat_id,
-                             text=mail.text_content)
-        
-
+                         text=mail)
 
 def main():
     # Create the EventHandler and pass it your bot's token.
diff --git a/utils/client.py b/utils/client.py
new file mode 100644
index 0000000..07518bb
--- /dev/null
+++ b/utils/client.py
@@ -0,0 +1,63 @@
+import logging
+import poplib
+from utils.mail import Email
+
+
+logger = logging.getLogger(__name__)
+
+
+
+class EmailClient(object):
+    def __init__(self, email_account, passwd):
+        self.email_account = email_account
+        self.password = passwd
+        self.server = self.connect(self)
+
+    @staticmethod
+    def connect(self):
+        # parse the server's hostname from email account
+        pop3_server = 'pop.'+self.email_account.split('@')[-1]
+        server = poplib.POP3_SSL(pop3_server)
+        # display the welcome info received from server,
+        # indicating the connection is set up properly
+        logger.info(server.getwelcome().decode('utf8'))
+        # authenticating
+        server.user(self.email_account)
+        server.pass_(self.password)
+        return server
+
+    def get_mails_list(self):
+        _, mails, _ = self.server.list()
+        return mails
+
+    def get_mails_count(self):
+        mails = self.get_mails_list()
+        return len(mails)
+
+    def get_mail_by_index(self, index):
+        resp_status, mail_lines, mail_octets = self.server.retr(index)
+        return Email(mail_lines)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is None:
+            print('exited normally\n')
+            self.server.quit()
+        else:
+            print('raise an exception! ' + str(exc_type))
+            self.server.close()
+            return False # Propagate
+
+
+
+if __name__ == '__main__':
+    useraccount = "XXXXX"
+    password = "XXXXXX"
+
+    client = EmailClient(useraccount, password)
+    num = client.get_mails_count()
+    print(num)
+    for i in range(1, num):
+        print(client.get_mail_by_index(i))
\ No newline at end of file
diff --git a/utils/mail.py b/utils/mail.py
index 2ead9ae..491faa8 100644
--- a/utils/mail.py
+++ b/utils/mail.py
@@ -1,149 +1,29 @@
-import base64
-from datetime import datetime
-from email.parser import Parser
-import logging
-import poplib
-import pytz
-import re
-
-
-logger = logging.getLogger(__name__)
-
-class MailDetails(object):
-    def __init__(self):
-        self.from_nickname = ""
-        self.from_account  = ""
-        self.to_nickname = ""
-        self.to_account = ""
-        self.subject = ""
-        self.receivedtime = ""
-        self.text_content = ""
-        self.html_content = ""
-
-
-def decode_byte(bstr, charset='utf8'):
-    return bstr.decode(charset)
-
-def get_rawcontent_charset(rawcontent):
-    for item in rawcontent:
-        if decode_byte(item).find('charset='):
-            charset = re.findall(re.compile('charset="(.*)"'), decode_byte(item))
-            for member in charset:
-                if member is not None:
-                    return member
-
-
-def parse_raw_mail_data(raw_lines, charset='utf8'):
-    msg_content = b'\r\n'.join(raw_lines).decode(encoding=charset)
-    return Parser().parsestr(text=msg_content)
-
-def decode_base64(s, charset='utf8'):
-    return str(base64.decodebytes(s.encode(encoding=charset)), encoding=charset)
-
-
-def get_mail_info(s):
-    try:
-        nickname, account = s.split(" ")
-    except ValueError:
-        nickname = ''
-        account = s
-
-    account = account.lstrip('<')
-    account = account.rstrip('>')
-    return nickname, account
-
-def get_mail_details(msg):
-    maildetails = MailDetails()
-
-    fromstr = msg.get('From')
-    from_nickname, from_account = get_mail_info(fromstr)
-    maildetails.from_nickname = from_nickname
-    maildetails.from_account = from_account
-    tostr = msg.get('To')
-    to_nickname, to_account = get_mail_info(tostr)
-    maildetails.to_nickname = to_nickname
-    maildetails.to_account = to_account
-
-    subject = msg.get('Subject')
-    try:
-        maildetails.subject = decode_base64(subject.split("?")[3], charset=subject.split("?")[1])
-    except IndexError:
-        maildetails.subject = subject
-    received_time = msg.get("Date")
-    maildetails.receivedtime = received_time
-
-    parts = msg.get_payload()
-    content_charset = parts[0].get_content_charset()
-    content = parts[0].as_string().split('base64')[-1]
-    try:
-        maildetails.text_content = decode_base64(content, content_charset)
-    except Exception as e:
-        logger.error('Exception caught: "%s"', e)
-        maildetails.text_content = content
-    content = parts[1].as_string().split('base64')[-1]
-    maildetails.html_content = content
-
-    return maildetails
-
-class EmailClient(object):
-    def __init__(self, email_account, passwd):
-        self.email_account = email_account
-        self.password = passwd
-        self.server = self.connect(self)
-
-    @staticmethod
-    def connect(self):
-        # parse the server's hostname from email account
-        pop3_server = 'pop.'+self.email_account.split('@')[-1]
-
-        server = poplib.POP3_SSL(pop3_server)
-
-        # display the welcome info received from server,
-        # indicating the connection is set up properly
-        print(server.getwelcome().decode('utf8'))
-
-        # authenticating
-        server.user(self.email_account)
-        server.pass_(self.password)
-
-        return server
-
-    def get_mails_list(self):
-        _, mails, _ = self.server.list()
-        return mails
-
-    def get_mails_count(self):
-        mails = self.get_mails_list()
-        return len(mails)
-
-
-
-    def get_mail_by_index(self, index):
-        resp_status, mail_lines, mail_octets = self.server.retr(index)
-        content_charset = get_rawcontent_charset(mail_lines)
-        data = parse_raw_mail_data(mail_lines, charset=content_charset or 'utf-8')
-        maildetails = get_mail_details(data)
-        return maildetails
-
-
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_type is None:
-            print('exited normally\n')
-            self.server.quit()
-        else:
-            print('raise an exception! ' + str(exc_type))
-            self.server.close()
-            return False # Propagate
-
-
-
-if __name__ == '__main__':
-    useraccount = "XXXXX"
-    password = "XXXXXX"
-
-    client = EmailClient(useraccount, password)
-    client.get_mails_count()
\ No newline at end of file
+from pyzmail import PyzMessage, decode_text
+
+class Email(object):
+    def __init__(self, raw_mail_lines):
+        msg_content = b'\r\n'.join(raw_mail_lines)
+        msg =  PyzMessage.factory(msg_content)
+
+        self.subject = msg.get_subject()
+        self.sender = msg.get_address('from')
+        self.date = msg.get_decoded_header('date', '')
+        self.id = msg.get_decoded_header('message-id', '')
+
+        for mailpart in msg.mailparts:
+            if mailpart.is_body=='text/plain':
+                payload, used_charset=decode_text(mailpart.get_payload(), mailpart.charset, None)
+                self.charset = used_charset
+                self.text = payload
+                return
+            else:
+                self.text = None
+
+    def __repr__(self):
+        mail_str = "Subject: %s\n" % self.subject
+        mail_str += "From: %s %s\n" % self.sender
+        mail_str += "Date: %s\n" % self.date
+        mail_str += "ID: %s\n" % self.id
+        if self.text:
+            mail_str += "Text: %s\n" % self.text
+        return mail_str