From 94b0f7bcbf4f587246f52ded900e896df3d92a90 Mon Sep 17 00:00:00 2001 From: Plamen Dimitrov Date: Thu, 12 Jul 2012 17:41:54 +0200 Subject: [PATCH] Rename main module to imap_fix_internaldate including header comments --- src/fix_imap_internaldate.py | 207 ------------------------------------------ src/imap_fix_internaldate.py | 207 ++++++++++++++++++++++++++++++++++++++++++ src/unit_tester.py | 1 - 3 files changed, 207 insertions(+), 208 deletions(-) delete mode 100644 src/fix_imap_internaldate.py create mode 100644 src/imap_fix_internaldate.py diff --git a/src/fix_imap_internaldate.py b/src/fix_imap_internaldate.py deleted file mode 100644 index 30ae4ff..0000000 --- a/src/fix_imap_internaldate.py +++ /dev/null @@ -1,207 +0,0 @@ -''' -fix_imap_internaldate.py - Fix the INTERNALDATE field on IMAP servers - -Copyright (c) 2012 Intra2net AG -Author: Plamen Dimitrov and Thomas Jarosch - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. -''' - -import sys -import csv -import argparse - -try: - import configparser -except ImportError: - import ConfigParser as configparser - -import logging -from mail_date_parser import MailDateParser -from mail_iterator import MailIterator -from caching_data import CachingData - -CONFIG_FILENAME = "fix_imap_internaldate.cfg" -LOG_FILENAME = "fix_imap_internaldate.log" -CSV_FILENAME = "userdata.csv" - -def main(): - """Interprets command arguments and initializes configuration and logger. - Then begins mail synchronization.""" - - # parse arguments - parser = argparse.ArgumentParser(description="Fix the INTERNALDATE field on IMAP servers. " - "Small tool to fix the IMAP internaldate " - "in case it's too much off compared to the last date " - "stored in the received lines.") - parser.add_argument('-u', '--update', dest='test_mode', action='store_false', - default=True, help='update all e-mails and exit test mode') - - # config and logging setup - config = load_configuration() - prepare_logger(config) - args = parser.parse_args() - - # check for update mode in the config - if config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing'): - args.test_mode = False - - if(args.test_mode): - logging.info("Testing mode initiated. No message will be modified on the server.") - else: - logging.info("Update mode initiated. Messages will be modified.") - - # proceed to main functionality - try: - synchronize_csv(config, args.test_mode) - except KeyboardInterrupt: - logging.info("Script was interrupted by the user.") - - logging.info("All done. Exiting.") - return - -def load_configuration(): - """Loads the script configuration from a file or creates such.""" - config = configparser.RawConfigParser() - success = config.read(CONFIG_FILENAME) - - # if no file is found create a default one - if(len(success)==0): - if(not config.has_section('basic_settings')): - config.add_section('basic_settings') - config.set('basic_settings', 'file_log_level', logging.INFO) - config.set('basic_settings', 'console_log_level', logging.INFO) - config.set('basic_settings', 'imap_server', 'imap.company.com') - config.set('basic_settings', 'tolerance_mins', 30) - config.set('basic_settings', 'skip_shared_folders', "True") - config.set('basic_settings', 'fallback_to_date_header', "False") - config.set('basic_settings', 'update_mode_i_know_what_i_am_doing', "False") - with open(CONFIG_FILENAME, 'w') as configfile: - config.write(configfile) - configfile.write("# 0 NOTSET, 10 DEBUG, 20 INFO, 30 WARNING, 40 ERROR, 50 CRITICAL\n") - print("Created initial config %s" % CONFIG_FILENAME) - - try: - config.get('basic_settings', 'file_log_level') - config.get('basic_settings', 'console_log_level') - config.get('basic_settings', 'imap_server') - config.getint('basic_settings', 'tolerance_mins') - config.getboolean('basic_settings', 'skip_shared_folders') - config.getboolean('basic_settings', 'fallback_to_date_header') - config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing') - except (configparser.NoSectionError, configparser.NoOptionError, ValueError) as ex: - print("Could not read config file '%s': %s." % (CONFIG_FILENAME, ex)) - print("Please change or remove the config file.") - sys.exit() - - return config - -def prepare_logger(config): - """Sets up the logging functionality""" - - # reset the log - with open(LOG_FILENAME, 'w'): - pass - - # add basic configuration - logging.basicConfig(filename=LOG_FILENAME, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - level=config.getint('basic_settings', 'file_log_level')) - - # add a handler for a console output - console = logging.StreamHandler() - console.setLevel(config.getint('basic_settings', 'console_log_level')) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - console.setFormatter(formatter) - logging.getLogger('').addHandler(console) - return - -def synchronize_csv(config, test_mode): - """Iterates through csv list of users and synchronizes their messages.""" - - # initialize loop permanent data - caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header')) - date_parser = MailDateParser() - server = config.get('basic_settings', 'imap_server') - tolerance = config.getint('basic_settings', 'tolerance_mins') * 60 - skip_shared_folders = config.getboolean('basic_settings', 'skip_shared_folders') - - # iterate through the users in the csv data - user_reader = csv.DictReader(open(CSV_FILENAME, "r"), delimiter=',') - for user in user_reader: - try: - session = MailIterator(server, user['username'], user['password'], skip_shared_folders) - except UserWarning as ex: - logging.error(ex) - continue - for mailbox in session: - try: - box = caching_data.retrieve_cached_mailbox(mailbox[0], mailbox[1], user['username']) - mail_ids = session.fetch_messages() - new_ids = box.synchronize(mail_ids, tolerance) - logging.info("%s new messages out of %s found in %s.", len(new_ids), len(mail_ids), box.name) - except UserWarning as ex: - logging.error(ex) - continue - for mid in new_ids: - try: - fetched_internal_date = session.fetch_internal_date(mid) - internal_date = date_parser.extract_internal_date(fetched_internal_date) - fetched_correct_date = session.fetch_received_date(mid) - correct_date = date_parser.extract_received_date(fetched_correct_date) - # check for empty received headers - if(correct_date == ""): - logging.debug("No received date could be found in message uid: %s - mailbox: %s - user: %s.", - mid.decode('iso-8859-1'), box.name, box.owner) - box.no_received_field += 1 - # correct these messages if required and override received_date from basic date - if(config.getboolean('basic_settings', 'fallback_to_date_header')): - fetched_correct_date = session.fetch_basic_date(mid) - correct_date = date_parser.extract_received_date(fetched_correct_date) - if(correct_date == ""): - logging.debug("No fallback date header could be found in message uid: %s - mailbox: %s - user: %s.", - mid.decode('iso-8859-1'), box.name, box.owner) - - if(correct_date == ""): - # skip synchronization for this message - continue - else: - # preserve only the first received line as fetched if everything is ok - fetched_correct_date = fetched_correct_date.split("Received:")[1] - except UserWarning as ex: - logging.error(ex) - continue - if(date_parser.compare_dates(correct_date, internal_date, tolerance)): - logging.warning("Date conflict found in message uid: %s - mailbox: %s - user: %s.\nInternal date %s is different from extracted date %s from header:\n%s.", - mid.decode('iso-8859-1'), box.name, box.owner, - internal_date.strftime("%d %b %Y %H:%M:%S"), - correct_date.strftime("%d %b %Y %H:%M:%S"), - fetched_correct_date) - if(not test_mode): - try: - session.update_message(mid, box.name, correct_date) - except UserWarning as ex: - logging.error(ex) - continue - - # count total emails for every user and mailbox - box.date_conflicts += 1 - - # if all messages were successfully fixed confirm caching - if(not test_mode): - box.confirm_change() - - # final report on date conflicts - caching_data.report_conflicts() - return - -if(__name__ == "__main__"): - main() diff --git a/src/imap_fix_internaldate.py b/src/imap_fix_internaldate.py new file mode 100644 index 0000000..1d9d757 --- /dev/null +++ b/src/imap_fix_internaldate.py @@ -0,0 +1,207 @@ +''' +imap_fix_internaldate.py - Fix the INTERNALDATE field on IMAP servers + +Copyright (c) 2012 Intra2net AG +Author: Plamen Dimitrov and Thomas Jarosch + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. +''' + +import sys +import csv +import argparse + +try: + import configparser +except ImportError: + import ConfigParser as configparser + +import logging +from mail_date_parser import MailDateParser +from mail_iterator import MailIterator +from caching_data import CachingData + +CONFIG_FILENAME = "fix_imap_internaldate.cfg" +LOG_FILENAME = "fix_imap_internaldate.log" +CSV_FILENAME = "userdata.csv" + +def main(): + """Interprets command arguments and initializes configuration and logger. + Then begins mail synchronization.""" + + # parse arguments + parser = argparse.ArgumentParser(description="Fix the INTERNALDATE field on IMAP servers. " + "Small tool to fix the IMAP internaldate " + "in case it's too much off compared to the last date " + "stored in the received lines.") + parser.add_argument('-u', '--update', dest='test_mode', action='store_false', + default=True, help='update all e-mails and exit test mode') + + # config and logging setup + config = load_configuration() + prepare_logger(config) + args = parser.parse_args() + + # check for update mode in the config + if config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing'): + args.test_mode = False + + if(args.test_mode): + logging.info("Testing mode initiated. No message will be modified on the server.") + else: + logging.info("Update mode initiated. Messages will be modified.") + + # proceed to main functionality + try: + synchronize_csv(config, args.test_mode) + except KeyboardInterrupt: + logging.info("Script was interrupted by the user.") + + logging.info("All done. Exiting.") + return + +def load_configuration(): + """Loads the script configuration from a file or creates such.""" + config = configparser.RawConfigParser() + success = config.read(CONFIG_FILENAME) + + # if no file is found create a default one + if(len(success)==0): + if(not config.has_section('basic_settings')): + config.add_section('basic_settings') + config.set('basic_settings', 'file_log_level', logging.INFO) + config.set('basic_settings', 'console_log_level', logging.INFO) + config.set('basic_settings', 'imap_server', 'imap.company.com') + config.set('basic_settings', 'tolerance_mins', 30) + config.set('basic_settings', 'skip_shared_folders', "True") + config.set('basic_settings', 'fallback_to_date_header', "False") + config.set('basic_settings', 'update_mode_i_know_what_i_am_doing', "False") + with open(CONFIG_FILENAME, 'w') as configfile: + config.write(configfile) + configfile.write("# 0 NOTSET, 10 DEBUG, 20 INFO, 30 WARNING, 40 ERROR, 50 CRITICAL\n") + print("Created initial config %s" % CONFIG_FILENAME) + + try: + config.get('basic_settings', 'file_log_level') + config.get('basic_settings', 'console_log_level') + config.get('basic_settings', 'imap_server') + config.getint('basic_settings', 'tolerance_mins') + config.getboolean('basic_settings', 'skip_shared_folders') + config.getboolean('basic_settings', 'fallback_to_date_header') + config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing') + except (configparser.NoSectionError, configparser.NoOptionError, ValueError) as ex: + print("Could not read config file '%s': %s." % (CONFIG_FILENAME, ex)) + print("Please change or remove the config file.") + sys.exit() + + return config + +def prepare_logger(config): + """Sets up the logging functionality""" + + # reset the log + with open(LOG_FILENAME, 'w'): + pass + + # add basic configuration + logging.basicConfig(filename=LOG_FILENAME, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + level=config.getint('basic_settings', 'file_log_level')) + + # add a handler for a console output + console = logging.StreamHandler() + console.setLevel(config.getint('basic_settings', 'console_log_level')) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) + return + +def synchronize_csv(config, test_mode): + """Iterates through csv list of users and synchronizes their messages.""" + + # initialize loop permanent data + caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header')) + date_parser = MailDateParser() + server = config.get('basic_settings', 'imap_server') + tolerance = config.getint('basic_settings', 'tolerance_mins') * 60 + skip_shared_folders = config.getboolean('basic_settings', 'skip_shared_folders') + + # iterate through the users in the csv data + user_reader = csv.DictReader(open(CSV_FILENAME, "r"), delimiter=',') + for user in user_reader: + try: + session = MailIterator(server, user['username'], user['password'], skip_shared_folders) + except UserWarning as ex: + logging.error(ex) + continue + for mailbox in session: + try: + box = caching_data.retrieve_cached_mailbox(mailbox[0], mailbox[1], user['username']) + mail_ids = session.fetch_messages() + new_ids = box.synchronize(mail_ids, tolerance) + logging.info("%s new messages out of %s found in %s.", len(new_ids), len(mail_ids), box.name) + except UserWarning as ex: + logging.error(ex) + continue + for mid in new_ids: + try: + fetched_internal_date = session.fetch_internal_date(mid) + internal_date = date_parser.extract_internal_date(fetched_internal_date) + fetched_correct_date = session.fetch_received_date(mid) + correct_date = date_parser.extract_received_date(fetched_correct_date) + # check for empty received headers + if(correct_date == ""): + logging.debug("No received date could be found in message uid: %s - mailbox: %s - user: %s.", + mid.decode('iso-8859-1'), box.name, box.owner) + box.no_received_field += 1 + # correct these messages if required and override received_date from basic date + if(config.getboolean('basic_settings', 'fallback_to_date_header')): + fetched_correct_date = session.fetch_basic_date(mid) + correct_date = date_parser.extract_received_date(fetched_correct_date) + if(correct_date == ""): + logging.debug("No fallback date header could be found in message uid: %s - mailbox: %s - user: %s.", + mid.decode('iso-8859-1'), box.name, box.owner) + + if(correct_date == ""): + # skip synchronization for this message + continue + else: + # preserve only the first received line as fetched if everything is ok + fetched_correct_date = fetched_correct_date.split("Received:")[1] + except UserWarning as ex: + logging.error(ex) + continue + if(date_parser.compare_dates(correct_date, internal_date, tolerance)): + logging.warning("Date conflict found in message uid: %s - mailbox: %s - user: %s.\nInternal date %s is different from extracted date %s from header:\n%s.", + mid.decode('iso-8859-1'), box.name, box.owner, + internal_date.strftime("%d %b %Y %H:%M:%S"), + correct_date.strftime("%d %b %Y %H:%M:%S"), + fetched_correct_date) + if(not test_mode): + try: + session.update_message(mid, box.name, correct_date) + except UserWarning as ex: + logging.error(ex) + continue + + # count total emails for every user and mailbox + box.date_conflicts += 1 + + # if all messages were successfully fixed confirm caching + if(not test_mode): + box.confirm_change() + + # final report on date conflicts + caching_data.report_conflicts() + return + +if(__name__ == "__main__"): + main() diff --git a/src/unit_tester.py b/src/unit_tester.py index ccd6eb2..4333766 100644 --- a/src/unit_tester.py +++ b/src/unit_tester.py @@ -45,7 +45,6 @@ class DateParseTester(unittest.TestCase): date = "11 Dec 2007 \r\n18:24:35 +0100" extracted_date = self.date_interp.extract_received_date(date) self.assertEqual(extracted_date, self.true_date, "Failed date format 2") - return def test_received_date_extraction3(self): """Tests the date extraction method.""" -- 1.7.1