from mailbox_state import MailboxState
CACHE_FILENAME = "message_cache.dat"
-CACHE_VERSION = 1
+CACHE_VERSION = "1"
class CachingData:
"""This class is responsible for the caching of data."""
# class attributes
# integer for version of the cache
version = None
+ # boolean flag which indicates fallback mode of the cache
+ fallback_to_date_header = None
# dictionary of usernames as keys and dictionaries as values
# the second dictionaries have unique mailbox keys and mailboxes as values
data = None
- def __init__(self):
+ def __init__(self, fallback_mode):
# open data file or create one and initialize date if not found
try:
cachefile = open(CACHE_FILENAME, 'rb')
- self.version, self.data = pickle.load(cachefile)
+ cache_info, self.data = pickle.load(cachefile)
+ cache_info = cache_info.split(' ')
+ self.version = cache_info[0]
if(self.version != CACHE_VERSION):
logging.warning("Cache file has version %s and the script version is %s. Deleting cache.",
self.version, CACHE_VERSION)
raise IOError
+ self.fallback_to_date_header = cache_info[1]
+ if(self.fallback_to_date_header != str(fallback_mode)):
+ logging.warning("Cache file date fallback mode setting is different than current settings. Deleting cache.")
+ raise IOError
logging.info("Cache file %s loaded", CACHE_FILENAME)
logging.debug("%s users found.", len(self.data))
except IOError:
self.version = CACHE_VERSION
+ stored_cache_info = self.version + ' ' + str(fallback_mode)
self.data = {}
with open(CACHE_FILENAME, 'wb') as cachefile:
- pickle.dump((self.version, self.data), cachefile)
+ pickle.dump((stored_cache_info, self.data), cachefile)
def __del__(self):
# create temporary file first
return
# serialize in file
- pickle.dump((self.version, self.data), cachefile)
+ stored_cache_info = self.version + ' ' + self.fallback_to_date_header
+ pickle.dump((stored_cache_info, self.data), cachefile)
logging.debug("%s users stored.", len(self.data))
# handle windows non-atomic rename
config = configparser.RawConfigParser()
success = config.read(CONFIG_FILENAME)
try:
- config.get('basic_settings', 'file_log_level')
- config.get('basic_settings', 'console_log_level')
- config.get('basic_settings', 'imap_server')
- config.getint('basic_settings', 'tolerance_mins')
- config.getboolean('basic_settings', 'skip_shared_folders')
- config.getboolean('basic_settings', 'fallback_to_date_header')
+ config.get('basic_settings', 'file_log_level')
+ config.get('basic_settings', 'console_log_level')
+ config.get('basic_settings', 'imap_server')
+ config.getint('basic_settings', 'tolerance_mins')
+ config.getboolean('basic_settings', 'skip_shared_folders')
+ config.getboolean('basic_settings', 'fallback_to_date_header')
except configparser.NoSectionError:
success = []
+ except configparser.NoOptionError:
+ success = []
except ValueError:
success = []
"""Iterates through csv list of users and synchronizes their messages."""
# initialize loop permanent data
- caching_data = CachingData()
+ caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header'))
date_parser = MailDateParser()
server = config.get('basic_settings', 'imap_server')
tolerance = config.getint('basic_settings', 'tolerance_mins') * 60
@classmethod
def compare_dates(cls, date1, date2, tolerance=1800):
- """Compares datetime objects for deviation given certain tolerance."""
- """Returns 1 if there is a significant difference."""
+ """Compares datetime objects for deviation given certain tolerance.
+ Returns 1 if there is a significant difference."""
logging.debug("Comparing dates %s <> %s.", date1, date2)
timedelta = abs(date1 - date2)
if(timedelta.total_seconds()>tolerance):
raise UserWarning("Could not log in as user " + username + ".")
self.logged_in = True
try:
- result, self.mailboxes = self.mail_con.list()
+ _result, self.mailboxes = self.mail_con.list()
except:
raise UserWarning("Could not retrieve mailboxes for user " + username + ".")
self.skip_shared_folders = skip_shared_folders
def __del__(self):
"""Closes the connection and the user session."""
if(self.logged_in):
- try:
- self.mail_con.close()
- self.mail_con.logout()
- except:
- pass
+ self.mail_con.close()
+ self.mail_con.logout()
def __iter__(self):
"""Iterates through all mailboxes, returns (uidval,name)."""
continue
# retrieve uidvalidity
try:
- result, data = self.mail_con.status(mailbox[2], '(UIDVALIDITY)')
+ _result, data = self.mail_con.status(mailbox[2], '(UIDVALIDITY)')
except:
raise UserWarning("Could not retrieve mailbox uidvalidity.")
uidval = UIDVAL_RESP.match(data[0].decode('iso-8859-1')).groups()
def fetch_messages(self):
"""Fetches the messages from the current mailbox, return list of uids."""
try:
- result, data = self.mail_con.uid('search', None, "ALL")
+ _result, data = self.mail_con.uid('search', None, "ALL")
except:
raise UserWarning("Could not fetch messages.")
mailid_list = data[0].split()
def fetch_internal_date(self, mid):
"""Fetches the internal date of a message, returns a time tuple."""
try:
- result, data = self.mail_con.uid('fetch', mid, '(INTERNALDATE)')
+ _result, data = self.mail_con.uid('fetch', mid, '(INTERNALDATE)')
except:
raise UserWarning("Could not fetch the internal date of message " + mid.decode('iso-8859-1') + ".")
internal_date = imaplib.Internaldate2tuple(data[0])
def fetch_received_date(self, mid):
"""Fetches the received date of a message, returns bytes reponse."""
try:
- result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (RECEIVED)])')
+ _result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (RECEIVED)])')
except:
raise UserWarning("Could not fetch the received header of message " + mid.decode('iso-8859-1') + ".")
return data[0][1].decode('iso-8859-1')
def fetch_basic_date(self, mid):
"""Fetches the basic date of a message, returns bytes reponse."""
try:
- result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (DATE)])')
+ _result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (DATE)])')
except:
raise UserWarning("Could not fetch the date header of message " + mid.decode('iso-8859-1') + ".")
return data[0][1].decode('iso-8859-1')
# retrieve and select flags to upload
fetched_flags = self.mail_con.uid('fetch', mid, '(FLAGS)')[1][0]
parsed_flags = imaplib.ParseFlags(fetched_flags)
- try:
- selected_flags = ()
- for flag in parsed_flags:
- if(flag != b'\\Recent'):
- selected_flags += (flag,)
- logging.debug("Selected flags %s from parsed flags %s.", selected_flags, parsed_flags)
- except Exception as ex:
- logging.error(ex)
+ selected_flags = ()
+ for flag in parsed_flags:
+ if(flag != b'\\Recent'):
+ selected_flags += (flag,)
+ logging.debug("Selected flags %s from parsed flags %s.", selected_flags, parsed_flags)
flags_str = " ".join(flag.decode('iso-8859-1') for flag in selected_flags)
# upload message copy and delete old one
del changed_dict['no_received_field']
return changed_dict
- def __setstate__(self, dict):
+ def __setstate__(self, state):
"""Prepares the MailboxState instance for unpickling."""
- self.name = dict["name"]
- self.uidvalidity = dict["uidvalidity"]
- self.owner = dict["owner"]
+ self.name = state["name"]
+ self.uidvalidity = state["uidvalidity"]
+ self.owner = state["owner"]
- self.uids = dict["uids"]
- self.tolerance = dict["tolerance"]
+ self.uids = state["uids"]
+ self.tolerance = state["tolerance"]
self.needs_save = False
self.date_conflicts = 0
self.no_received_field = 0
- self.key = dict["key"]
+ self.key = state["key"]
return
'''
import unittest
-import datetime, date_interpreter
+import datetime
+from mail_date_parser import MailDateParser
class MailScriptTester(unittest.TestCase):
+ """Used to test the date retrievel and the MailDateParser class."""
# class attributes
# DateInterpreter instance testing the DateInterpreter methods
- date_interp = None
+ date_interp = MailDateParser()
# datetime for comparison with extracted datetimes and assertions
- true_date = None
+ true_date = datetime.datetime(2007, 12, 11, 18, 24, 35)
- def setUp(self):
- self.date_interp = date_interpreter.DateInterpreter()
- self.true_date = datetime.datetime(2007, 12, 11, 18, 24, 35)
+ def set_up(self):
+ """Prepares the testing confitions."""
+ # tester has problems if object is set to none so
+ # attributes are prepared above
def test_received_date_extraction1(self):
"""Tests the date extraction method."""
- date = [[0, b"Tue, 11 Dec 2007 18:24:35 +0100"]]
+ date = "Tue, 11 Dec 2007 18:24:35 +0100"
extracted_date = self.date_interp.extract_received_date(date)
self.assertEqual(extracted_date, self.true_date, "Failed date format 1")
def test_received_date_extraction2(self):
"""Tests the date extraction method."""
- date = [[0, b"11 Dec 2007 \r\n18:24:35 +0100"]]
+ date = "11 Dec 2007 \r\n18:24:35 +0100"
extracted_date = self.date_interp.extract_received_date(date)
self.assertEqual(extracted_date, self.true_date, "Failed date format 2")
return
def test_received_date_extraction3(self):
"""Tests the date extraction method."""
- date = [[0, b"11 Dec 2007 18:24:35 +0100"]]
+ date = "11 Dec 2007 18:24:35 +0100"
extracted_date = self.date_interp.extract_received_date(date)
self.assertEqual(extracted_date, self.true_date, "Failed date format 3")
def test_received_date_extraction4(self):
"""Tests the date extraction method."""
- date = [[0, b"11 Dec 2007 18:24:35"]]
+ date = "11 Dec 2007 18:24:35"
extracted_date = self.date_interp.extract_received_date(date)
#should not be equal because of time zone assumption
self.assertNotEqual(extracted_date, self.true_date, "Failed date format 4")
def test_received_date_extraction5(self):
"""Tests the received date extraction method."""
- date = [[0, b"11 Dec 2007 18:24:35 GMT"]]
+ date = "11 Dec 2007 18:24:35 GMT"
extracted_date = self.date_interp.extract_received_date(date)
#should not be equal because of time zone assumption
self.assertNotEqual(extracted_date, self.true_date, "Failed date format 5")
def test_received_date_extraction6(self):
"""Tests the received date extraction method."""
- date = [[0, b'Received: from intranator.m.i2n ([unix socket])'
- b'by intranator.m.i2n with LMTPA; Tue, 11 Dec 2007 18:24:35'
- b'+0100Received: from localhost (intranator.m.i2n [127.0.0.1])'
- b'by localhost (Postfix) with ESMTP id 895812AC54for <intra2net_thomas@intranator.m.i2n>;'
- b'Sun, 13 Mar 2011 18:47:18 +0100 (CET)Received: from re04.intra2net.com '
- b'(re04.intra2net.com [82.165.46.26])(using TLSv1 with cipher ADH-AES256-SHA '
- b'(256/256 bits))(No client certificate requested)by intranator.m.i2n (Postfix) with '
- b'ESMTPS id 28DB92AC53for <thomas.jarosch@intra2net.com>; Sun, 13 Mar 2011 18:47:15 +0100 '
- b'(CET)Received: from postfix.charite.de (postfix.charite.de [141.42.206.35])(using TLSv1 '
- b'with cipher ADH-AES256-SHA (256/256 bits))(No client certificate requested)by '
- b're04.intra2net.com (Postfix) with ESMTP id C054A3010Afor <thomas.jarosch@intra2net.com>; '
- b'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from localhost (localhost [127.0.0.1])by '
- b'de.postfix.org (Postfix) with ESMTP id 7FCCFF7879for <thomas.jarosch@intra2net.com>; '
- b'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from de.postfix.org ([127.0.0.1])by '
- b'localhost (de.postfix.org [127.0.0.1]) (amavisd-new, port 10026)with LMTP id '
- b'YSXF-vf3+6E1 for <thomas.jarosch@intra2net.com>;Sun, 13 Mar 2011 18:47:14 +0100 (CET)'
- b'Received: from de.postfix.org (localhost [127.0.0.1])by de.postfix.org (Postfix) with '
- b'ESMTP id 3C3123DF1Efor <thomas.jarosch@intra2net.com>; Sun, 13 Mar 2011 18:46:33 +0100 '
- b'(CET)Received: from localhost (localhost [127.0.0.1])by de.postfix.org (Postfix) with '
- b'ESMTP id AB6CE3DBD2for <amavis-users@amavis.org>; Sun, 13 Mar 2011 18:45:57 +0100 (CET)'
- b'Received: from de.postfix.org ([127.0.0.1])by localhost (de.postfix.org [127.0.0.1]) '
- b'(amavisd-new, port 10024)with ESMTP id mBYiZO8wREeS for <amavis-users@amavis.org>;Sun, '
- b'13 Mar 2011 18:45:56 +0100 (CET)Received: from mail.inetmsg.com (mail.inetmsg.com '
- b'[173.10.94.185])by de.postfix.org (Postfix) with ESMTPSfor <amavis-users@amavis.org>; '
- b'Sun, 13 Mar 2011 18:45:55 +0100 (CET)Received: from [192.168.1.107] (fw1.inetmsg.com '
- b'[10.20.30.253])(using TLSv1 with cipher DHE-RSA-CAMELLIA256-SHA (256/256 bits))'
- b'(No client certificate requested)by mail.inetmsg.com (INetMsg Mail Service) with ESMTPSA '
- b'id 0B95326CD1for <amavis-users@amavis.org>; Sun, 13 Mar 2011 10:45:41 -0700 (PDT)"]]']]
+ date = 'Received: from intranator.m.i2n ([unix socket])'\
+ 'by intranator.m.i2n with LMTPA; Tue, 11 Dec 2007 18:24:35'\
+ '+0100Received: from localhost (intranator.m.i2n [127.0.0.1])'\
+ 'by localhost (Postfix) with ESMTP id 895812AC54for <intra2net_thomas@intranator.m.i2n>;'\
+ 'Sun, 13 Mar 2011 18:47:18 +0100 (CET)Received: from re04.intra2net.com '\
+ '(re04.intra2net.com [82.165.46.26])(using TLSv1 with cipher ADH-AES256-SHA '\
+ '(256/256 bits))(No client certificate requested)by intranator.m.i2n (Postfix) with '\
+ 'ESMTPS id 28DB92AC53for <thomas.jarosch@intra2net.com>; Sun, 13 Mar 2011 18:47:15 +0100 '\
+ '(CET)Received: from postfix.charite.de (postfix.charite.de [141.42.206.35])(using TLSv1 '\
+ 'with cipher ADH-AES256-SHA (256/256 bits))(No client certificate requested)by '\
+ 're04.intra2net.com (Postfix) with ESMTP id C054A3010Afor <thomas.jarosch@intra2net.com>; '\
+ 'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from localhost (localhost [127.0.0.1])by '\
+ 'de.postfix.org (Postfix) with ESMTP id 7FCCFF7879for <thomas.jarosch@intra2net.com>; '\
+ 'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from de.postfix.org ([127.0.0.1])by '\
+ 'localhost (de.postfix.org [127.0.0.1]) (amavisd-new, port 10026)with LMTP id '\
+ 'YSXF-vf3+6E1 for <thomas.jarosch@intra2net.com>;Sun, 13 Mar 2011 18:47:14 +0100 (CET)'\
+ 'Received: from de.postfix.org (localhost [127.0.0.1])by de.postfix.org (Postfix) with '\
+ 'ESMTP id 3C3123DF1Efor <thomas.jarosch@intra2net.com>; Sun, 13 Mar 2011 18:46:33 +0100 '\
+ '(CET)Received: from localhost (localhost [127.0.0.1])by de.postfix.org (Postfix) with '\
+ 'ESMTP id AB6CE3DBD2for <amavis-users@amavis.org>; Sun, 13 Mar 2011 18:45:57 +0100 (CET)'\
+ 'Received: from de.postfix.org ([127.0.0.1])by localhost (de.postfix.org [127.0.0.1]) '\
+ '(amavisd-new, port 10024)with ESMTP id mBYiZO8wREeS for <amavis-users@amavis.org>;Sun, '\
+ '13 Mar 2011 18:45:56 +0100 (CET)Received: from mail.inetmsg.com (mail.inetmsg.com '\
+ '[173.10.94.185])by de.postfix.org (Postfix) with ESMTPSfor <amavis-users@amavis.org>; '\
+ 'Sun, 13 Mar 2011 18:45:55 +0100 (CET)Received: from [192.168.1.107] (fw1.inetmsg.com '\
+ '[10.20.30.253])(using TLSv1 with cipher DHE-RSA-CAMELLIA256-SHA (256/256 bits))'\
+ '(No client certificate requested)by mail.inetmsg.com (INetMsg Mail Service) with ESMTPSA '\
+ 'id 0B95326CD1for <amavis-users@amavis.org>; Sun, 13 Mar 2011 10:45:41 -0700 (PDT)"]]'
extracted_date = self.date_interp.extract_received_date(date)
#should not be equal because of time zone assumption
self.assertEqual(extracted_date, self.true_date, "Failed date format 6")
def test_compare_dates(self):
"""Tests the date comparison method."""
- self.true_date2 = datetime.datetime(2007, 12, 11, 18, 34, 35)
+ true_date2 = datetime.datetime(2007, 12, 11, 18, 34, 35)
#is difference of 10 mins significant if tolerance is 9 mins
- self.assertTrue(bool(self.date_interp.compare_dates(self.true_date, self.true_date2, 9*60)), "Failed at comparison test")
+ self.assertTrue(bool(self.date_interp.compare_dates(self.true_date, true_date2, 9*60)), "Failed at comparison test")
#is difference of 10 mins significant if tolerance is 11 mins
- self.assertFalse(bool(self.date_interp.compare_dates(self.true_date, self.true_date2, 11*60)), "Failed at comparison test")
+ self.assertFalse(bool(self.date_interp.compare_dates(self.true_date, true_date2, 11*60)), "Failed at comparison test")
if __name__ == '__main__':
unittest.main()
git://developer.intra2net.com / imap-fix-internaldate / commitdiff