From: Plamen Dimitrov Date: Wed, 4 Jul 2012 15:58:26 +0000 (+0200) Subject: Style improvements and cache from settings validation added X-Git-Url: http://developer.intra2net.com/git/?p=imap-fix-internaldate;a=commitdiff_plain;h=db3f09a6794a014e84d9940961660794217c6a52 Style improvements and cache from settings validation added --- diff --git a/src/caching_data.py b/src/caching_data.py index ff32212..4ee3ac0 100644 --- a/src/caching_data.py +++ b/src/caching_data.py @@ -20,7 +20,7 @@ import logging from mailbox_state import MailboxState CACHE_FILENAME = "message_cache.dat" -CACHE_VERSION = 1 +CACHE_VERSION = "1" class CachingData: """This class is responsible for the caching of data.""" @@ -28,26 +28,35 @@ class CachingData: # class attributes # integer for version of the cache version = None + # boolean flag which indicates fallback mode of the cache + fallback_to_date_header = None # dictionary of usernames as keys and dictionaries as values # the second dictionaries have unique mailbox keys and mailboxes as values data = None - def __init__(self): + def __init__(self, fallback_mode): # open data file or create one and initialize date if not found try: cachefile = open(CACHE_FILENAME, 'rb') - self.version, self.data = pickle.load(cachefile) + cache_info, self.data = pickle.load(cachefile) + cache_info = cache_info.split(' ') + self.version = cache_info[0] if(self.version != CACHE_VERSION): logging.warning("Cache file has version %s and the script version is %s. Deleting cache.", self.version, CACHE_VERSION) raise IOError + self.fallback_to_date_header = cache_info[1] + if(self.fallback_to_date_header != str(fallback_mode)): + logging.warning("Cache file date fallback mode setting is different than current settings. Deleting cache.") + raise IOError logging.info("Cache file %s loaded", CACHE_FILENAME) logging.debug("%s users found.", len(self.data)) except IOError: self.version = CACHE_VERSION + stored_cache_info = self.version + ' ' + str(fallback_mode) self.data = {} with open(CACHE_FILENAME, 'wb') as cachefile: - pickle.dump((self.version, self.data), cachefile) + pickle.dump((stored_cache_info, self.data), cachefile) def __del__(self): # create temporary file first @@ -75,7 +84,8 @@ class CachingData: return # serialize in file - pickle.dump((self.version, self.data), cachefile) + stored_cache_info = self.version + ' ' + self.fallback_to_date_header + pickle.dump((stored_cache_info, self.data), cachefile) logging.debug("%s users stored.", len(self.data)) # handle windows non-atomic rename diff --git a/src/fix_imap_internaldate.py b/src/fix_imap_internaldate.py index 7e97701..91f99bf 100644 --- a/src/fix_imap_internaldate.py +++ b/src/fix_imap_internaldate.py @@ -68,14 +68,16 @@ def load_configuration(): config = configparser.RawConfigParser() success = config.read(CONFIG_FILENAME) try: - config.get('basic_settings', 'file_log_level') - config.get('basic_settings', 'console_log_level') - config.get('basic_settings', 'imap_server') - config.getint('basic_settings', 'tolerance_mins') - config.getboolean('basic_settings', 'skip_shared_folders') - config.getboolean('basic_settings', 'fallback_to_date_header') + config.get('basic_settings', 'file_log_level') + config.get('basic_settings', 'console_log_level') + config.get('basic_settings', 'imap_server') + config.getint('basic_settings', 'tolerance_mins') + config.getboolean('basic_settings', 'skip_shared_folders') + config.getboolean('basic_settings', 'fallback_to_date_header') except configparser.NoSectionError: success = [] + except configparser.NoOptionError: + success = [] except ValueError: success = [] @@ -119,7 +121,7 @@ def synchronize_csv(config, test_mode): """Iterates through csv list of users and synchronizes their messages.""" # initialize loop permanent data - caching_data = CachingData() + caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header')) date_parser = MailDateParser() server = config.get('basic_settings', 'imap_server') tolerance = config.getint('basic_settings', 'tolerance_mins') * 60 diff --git a/src/mail_date_parser.py b/src/mail_date_parser.py index d957d23..5bfb6ca 100644 --- a/src/mail_date_parser.py +++ b/src/mail_date_parser.py @@ -90,8 +90,8 @@ class MailDateParser: @classmethod def compare_dates(cls, date1, date2, tolerance=1800): - """Compares datetime objects for deviation given certain tolerance.""" - """Returns 1 if there is a significant difference.""" + """Compares datetime objects for deviation given certain tolerance. + Returns 1 if there is a significant difference.""" logging.debug("Comparing dates %s <> %s.", date1, date2) timedelta = abs(date1 - date2) if(timedelta.total_seconds()>tolerance): diff --git a/src/mail_iterator.py b/src/mail_iterator.py index d30bed1..c1f2e80 100644 --- a/src/mail_iterator.py +++ b/src/mail_iterator.py @@ -49,7 +49,7 @@ class MailIterator: raise UserWarning("Could not log in as user " + username + ".") self.logged_in = True try: - result, self.mailboxes = self.mail_con.list() + _result, self.mailboxes = self.mail_con.list() except: raise UserWarning("Could not retrieve mailboxes for user " + username + ".") self.skip_shared_folders = skip_shared_folders @@ -57,11 +57,8 @@ class MailIterator: def __del__(self): """Closes the connection and the user session.""" if(self.logged_in): - try: - self.mail_con.close() - self.mail_con.logout() - except: - pass + self.mail_con.close() + self.mail_con.logout() def __iter__(self): """Iterates through all mailboxes, returns (uidval,name).""" @@ -74,7 +71,7 @@ class MailIterator: continue # retrieve uidvalidity try: - result, data = self.mail_con.status(mailbox[2], '(UIDVALIDITY)') + _result, data = self.mail_con.status(mailbox[2], '(UIDVALIDITY)') except: raise UserWarning("Could not retrieve mailbox uidvalidity.") uidval = UIDVAL_RESP.match(data[0].decode('iso-8859-1')).groups() @@ -90,7 +87,7 @@ class MailIterator: def fetch_messages(self): """Fetches the messages from the current mailbox, return list of uids.""" try: - result, data = self.mail_con.uid('search', None, "ALL") + _result, data = self.mail_con.uid('search', None, "ALL") except: raise UserWarning("Could not fetch messages.") mailid_list = data[0].split() @@ -99,7 +96,7 @@ class MailIterator: def fetch_internal_date(self, mid): """Fetches the internal date of a message, returns a time tuple.""" try: - result, data = self.mail_con.uid('fetch', mid, '(INTERNALDATE)') + _result, data = self.mail_con.uid('fetch', mid, '(INTERNALDATE)') except: raise UserWarning("Could not fetch the internal date of message " + mid.decode('iso-8859-1') + ".") internal_date = imaplib.Internaldate2tuple(data[0]) @@ -108,7 +105,7 @@ class MailIterator: def fetch_received_date(self, mid): """Fetches the received date of a message, returns bytes reponse.""" try: - result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (RECEIVED)])') + _result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (RECEIVED)])') except: raise UserWarning("Could not fetch the received header of message " + mid.decode('iso-8859-1') + ".") return data[0][1].decode('iso-8859-1') @@ -116,7 +113,7 @@ class MailIterator: def fetch_basic_date(self, mid): """Fetches the basic date of a message, returns bytes reponse.""" try: - result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (DATE)])') + _result, data = self.mail_con.uid('fetch', mid, '(BODY.PEEK[HEADER.FIELDS (DATE)])') except: raise UserWarning("Could not fetch the date header of message " + mid.decode('iso-8859-1') + ".") return data[0][1].decode('iso-8859-1') @@ -131,14 +128,11 @@ class MailIterator: # retrieve and select flags to upload fetched_flags = self.mail_con.uid('fetch', mid, '(FLAGS)')[1][0] parsed_flags = imaplib.ParseFlags(fetched_flags) - try: - selected_flags = () - for flag in parsed_flags: - if(flag != b'\\Recent'): - selected_flags += (flag,) - logging.debug("Selected flags %s from parsed flags %s.", selected_flags, parsed_flags) - except Exception as ex: - logging.error(ex) + selected_flags = () + for flag in parsed_flags: + if(flag != b'\\Recent'): + selected_flags += (flag,) + logging.debug("Selected flags %s from parsed flags %s.", selected_flags, parsed_flags) flags_str = " ".join(flag.decode('iso-8859-1') for flag in selected_flags) # upload message copy and delete old one diff --git a/src/mailbox_state.py b/src/mailbox_state.py index 2d1e3af..6d3cd23 100644 --- a/src/mailbox_state.py +++ b/src/mailbox_state.py @@ -64,20 +64,20 @@ class MailboxState: del changed_dict['no_received_field'] return changed_dict - def __setstate__(self, dict): + def __setstate__(self, state): """Prepares the MailboxState instance for unpickling.""" - self.name = dict["name"] - self.uidvalidity = dict["uidvalidity"] - self.owner = dict["owner"] + self.name = state["name"] + self.uidvalidity = state["uidvalidity"] + self.owner = state["owner"] - self.uids = dict["uids"] - self.tolerance = dict["tolerance"] + self.uids = state["uids"] + self.tolerance = state["tolerance"] self.needs_save = False self.date_conflicts = 0 self.no_received_field = 0 - self.key = dict["key"] + self.key = state["key"] return diff --git a/src/unit_tester.py b/src/unit_tester.py index 13da94b..56f6efc 100644 --- a/src/unit_tester.py +++ b/src/unit_tester.py @@ -16,94 +16,97 @@ GNU General Public License for more details. ''' import unittest -import datetime, date_interpreter +import datetime +from mail_date_parser import MailDateParser class MailScriptTester(unittest.TestCase): + """Used to test the date retrievel and the MailDateParser class.""" # class attributes # DateInterpreter instance testing the DateInterpreter methods - date_interp = None + date_interp = MailDateParser() # datetime for comparison with extracted datetimes and assertions - true_date = None + true_date = datetime.datetime(2007, 12, 11, 18, 24, 35) - def setUp(self): - self.date_interp = date_interpreter.DateInterpreter() - self.true_date = datetime.datetime(2007, 12, 11, 18, 24, 35) + def set_up(self): + """Prepares the testing confitions.""" + # tester has problems if object is set to none so + # attributes are prepared above def test_received_date_extraction1(self): """Tests the date extraction method.""" - date = [[0, b"Tue, 11 Dec 2007 18:24:35 +0100"]] + date = "Tue, 11 Dec 2007 18:24:35 +0100" extracted_date = self.date_interp.extract_received_date(date) self.assertEqual(extracted_date, self.true_date, "Failed date format 1") def test_received_date_extraction2(self): """Tests the date extraction method.""" - date = [[0, b"11 Dec 2007 \r\n18:24:35 +0100"]] + date = "11 Dec 2007 \r\n18:24:35 +0100" extracted_date = self.date_interp.extract_received_date(date) self.assertEqual(extracted_date, self.true_date, "Failed date format 2") return def test_received_date_extraction3(self): """Tests the date extraction method.""" - date = [[0, b"11 Dec 2007 18:24:35 +0100"]] + date = "11 Dec 2007 18:24:35 +0100" extracted_date = self.date_interp.extract_received_date(date) self.assertEqual(extracted_date, self.true_date, "Failed date format 3") def test_received_date_extraction4(self): """Tests the date extraction method.""" - date = [[0, b"11 Dec 2007 18:24:35"]] + date = "11 Dec 2007 18:24:35" extracted_date = self.date_interp.extract_received_date(date) #should not be equal because of time zone assumption self.assertNotEqual(extracted_date, self.true_date, "Failed date format 4") def test_received_date_extraction5(self): """Tests the received date extraction method.""" - date = [[0, b"11 Dec 2007 18:24:35 GMT"]] + date = "11 Dec 2007 18:24:35 GMT" extracted_date = self.date_interp.extract_received_date(date) #should not be equal because of time zone assumption self.assertNotEqual(extracted_date, self.true_date, "Failed date format 5") def test_received_date_extraction6(self): """Tests the received date extraction method.""" - date = [[0, b'Received: from intranator.m.i2n ([unix socket])' - b'by intranator.m.i2n with LMTPA; Tue, 11 Dec 2007 18:24:35' - b'+0100Received: from localhost (intranator.m.i2n [127.0.0.1])' - b'by localhost (Postfix) with ESMTP id 895812AC54for ;' - b'Sun, 13 Mar 2011 18:47:18 +0100 (CET)Received: from re04.intra2net.com ' - b'(re04.intra2net.com [82.165.46.26])(using TLSv1 with cipher ADH-AES256-SHA ' - b'(256/256 bits))(No client certificate requested)by intranator.m.i2n (Postfix) with ' - b'ESMTPS id 28DB92AC53for ; Sun, 13 Mar 2011 18:47:15 +0100 ' - b'(CET)Received: from postfix.charite.de (postfix.charite.de [141.42.206.35])(using TLSv1 ' - b'with cipher ADH-AES256-SHA (256/256 bits))(No client certificate requested)by ' - b're04.intra2net.com (Postfix) with ESMTP id C054A3010Afor ; ' - b'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from localhost (localhost [127.0.0.1])by ' - b'de.postfix.org (Postfix) with ESMTP id 7FCCFF7879for ; ' - b'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from de.postfix.org ([127.0.0.1])by ' - b'localhost (de.postfix.org [127.0.0.1]) (amavisd-new, port 10026)with LMTP id ' - b'YSXF-vf3+6E1 for ;Sun, 13 Mar 2011 18:47:14 +0100 (CET)' - b'Received: from de.postfix.org (localhost [127.0.0.1])by de.postfix.org (Postfix) with ' - b'ESMTP id 3C3123DF1Efor ; Sun, 13 Mar 2011 18:46:33 +0100 ' - b'(CET)Received: from localhost (localhost [127.0.0.1])by de.postfix.org (Postfix) with ' - b'ESMTP id AB6CE3DBD2for ; Sun, 13 Mar 2011 18:45:57 +0100 (CET)' - b'Received: from de.postfix.org ([127.0.0.1])by localhost (de.postfix.org [127.0.0.1]) ' - b'(amavisd-new, port 10024)with ESMTP id mBYiZO8wREeS for ;Sun, ' - b'13 Mar 2011 18:45:56 +0100 (CET)Received: from mail.inetmsg.com (mail.inetmsg.com ' - b'[173.10.94.185])by de.postfix.org (Postfix) with ESMTPSfor ; ' - b'Sun, 13 Mar 2011 18:45:55 +0100 (CET)Received: from [192.168.1.107] (fw1.inetmsg.com ' - b'[10.20.30.253])(using TLSv1 with cipher DHE-RSA-CAMELLIA256-SHA (256/256 bits))' - b'(No client certificate requested)by mail.inetmsg.com (INetMsg Mail Service) with ESMTPSA ' - b'id 0B95326CD1for ; Sun, 13 Mar 2011 10:45:41 -0700 (PDT)"]]']] + date = 'Received: from intranator.m.i2n ([unix socket])'\ + 'by intranator.m.i2n with LMTPA; Tue, 11 Dec 2007 18:24:35'\ + '+0100Received: from localhost (intranator.m.i2n [127.0.0.1])'\ + 'by localhost (Postfix) with ESMTP id 895812AC54for ;'\ + 'Sun, 13 Mar 2011 18:47:18 +0100 (CET)Received: from re04.intra2net.com '\ + '(re04.intra2net.com [82.165.46.26])(using TLSv1 with cipher ADH-AES256-SHA '\ + '(256/256 bits))(No client certificate requested)by intranator.m.i2n (Postfix) with '\ + 'ESMTPS id 28DB92AC53for ; Sun, 13 Mar 2011 18:47:15 +0100 '\ + '(CET)Received: from postfix.charite.de (postfix.charite.de [141.42.206.35])(using TLSv1 '\ + 'with cipher ADH-AES256-SHA (256/256 bits))(No client certificate requested)by '\ + 're04.intra2net.com (Postfix) with ESMTP id C054A3010Afor ; '\ + 'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from localhost (localhost [127.0.0.1])by '\ + 'de.postfix.org (Postfix) with ESMTP id 7FCCFF7879for ; '\ + 'Sun, 13 Mar 2011 18:47:14 +0100 (CET)Received: from de.postfix.org ([127.0.0.1])by '\ + 'localhost (de.postfix.org [127.0.0.1]) (amavisd-new, port 10026)with LMTP id '\ + 'YSXF-vf3+6E1 for ;Sun, 13 Mar 2011 18:47:14 +0100 (CET)'\ + 'Received: from de.postfix.org (localhost [127.0.0.1])by de.postfix.org (Postfix) with '\ + 'ESMTP id 3C3123DF1Efor ; Sun, 13 Mar 2011 18:46:33 +0100 '\ + '(CET)Received: from localhost (localhost [127.0.0.1])by de.postfix.org (Postfix) with '\ + 'ESMTP id AB6CE3DBD2for ; Sun, 13 Mar 2011 18:45:57 +0100 (CET)'\ + 'Received: from de.postfix.org ([127.0.0.1])by localhost (de.postfix.org [127.0.0.1]) '\ + '(amavisd-new, port 10024)with ESMTP id mBYiZO8wREeS for ;Sun, '\ + '13 Mar 2011 18:45:56 +0100 (CET)Received: from mail.inetmsg.com (mail.inetmsg.com '\ + '[173.10.94.185])by de.postfix.org (Postfix) with ESMTPSfor ; '\ + 'Sun, 13 Mar 2011 18:45:55 +0100 (CET)Received: from [192.168.1.107] (fw1.inetmsg.com '\ + '[10.20.30.253])(using TLSv1 with cipher DHE-RSA-CAMELLIA256-SHA (256/256 bits))'\ + '(No client certificate requested)by mail.inetmsg.com (INetMsg Mail Service) with ESMTPSA '\ + 'id 0B95326CD1for ; Sun, 13 Mar 2011 10:45:41 -0700 (PDT)"]]' extracted_date = self.date_interp.extract_received_date(date) #should not be equal because of time zone assumption self.assertEqual(extracted_date, self.true_date, "Failed date format 6") def test_compare_dates(self): """Tests the date comparison method.""" - self.true_date2 = datetime.datetime(2007, 12, 11, 18, 34, 35) + true_date2 = datetime.datetime(2007, 12, 11, 18, 34, 35) #is difference of 10 mins significant if tolerance is 9 mins - self.assertTrue(bool(self.date_interp.compare_dates(self.true_date, self.true_date2, 9*60)), "Failed at comparison test") + self.assertTrue(bool(self.date_interp.compare_dates(self.true_date, true_date2, 9*60)), "Failed at comparison test") #is difference of 10 mins significant if tolerance is 11 mins - self.assertFalse(bool(self.date_interp.compare_dates(self.true_date, self.true_date2, 11*60)), "Failed at comparison test") + self.assertFalse(bool(self.date_interp.compare_dates(self.true_date, true_date2, 11*60)), "Failed at comparison test") if __name__ == '__main__': unittest.main()