''' imap_fix_internaldate.py - Fix the INTERNALDATE field on IMAP servers Copyright (c) 2012 Intra2net AG Author: Plamen Dimitrov and Thomas Jarosch This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. ''' import sys import csv import argparse try: import configparser except ImportError: import ConfigParser as configparser import logging from mail_date_parser import MailDateParser from mail_iterator import MailIterator from caching_data import CachingData CONFIG_FILENAME = "fix_imap_internaldate.cfg" LOG_FILENAME = "fix_imap_internaldate.log" CSV_FILENAME = "userdata.csv" def main(): """Interprets command arguments and initializes configuration and logger. Then begins mail synchronization.""" # parse arguments parser = argparse.ArgumentParser(description="Fix the INTERNALDATE field on IMAP servers. " "Small tool to fix the IMAP internaldate " "in case it's too much off compared to the last date " "stored in the received lines.") parser.add_argument('-u', '--update', dest='test_mode', action='store_false', default=True, help='update all e-mails and exit test mode') # config and logging setup config = load_configuration() prepare_logger(config) args = parser.parse_args() # check for update mode in the config if config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing'): args.test_mode = False if(args.test_mode): logging.info("Testing mode initiated. No message will be modified on the server.") else: logging.info("Update mode initiated. Messages will be modified.") # proceed to main functionality try: synchronize_csv(config, args.test_mode) except KeyboardInterrupt: logging.info("Script was interrupted by the user.") logging.info("All done. Exiting.") return def load_configuration(): """Loads the script configuration from a file or creates such.""" config = configparser.RawConfigParser() success = config.read(CONFIG_FILENAME) # if no file is found create a default one if(len(success)==0): if(not config.has_section('basic_settings')): config.add_section('basic_settings') config.set('basic_settings', 'file_log_level', logging.INFO) config.set('basic_settings', 'console_log_level', logging.INFO) config.set('basic_settings', 'imap_server', 'imap.company.com') config.set('basic_settings', 'tolerance_mins', 30) config.set('basic_settings', 'skip_shared_folders', "True") config.set('basic_settings', 'fallback_to_date_header', "False") config.set('basic_settings', 'update_mode_i_know_what_i_am_doing', "False") with open(CONFIG_FILENAME, 'w') as configfile: config.write(configfile) configfile.write("# 0 NOTSET, 10 DEBUG, 20 INFO, 30 WARNING, 40 ERROR, 50 CRITICAL\n") print("Created initial config %s" % CONFIG_FILENAME) try: config.get('basic_settings', 'file_log_level') config.get('basic_settings', 'console_log_level') config.get('basic_settings', 'imap_server') config.getint('basic_settings', 'tolerance_mins') config.getboolean('basic_settings', 'skip_shared_folders') config.getboolean('basic_settings', 'fallback_to_date_header') config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing') except (configparser.NoSectionError, configparser.NoOptionError, ValueError) as ex: print("Could not read config file '%s': %s." % (CONFIG_FILENAME, ex)) print("Please change or remove the config file.") sys.exit() return config def prepare_logger(config): """Sets up the logging functionality""" # reset the log with open(LOG_FILENAME, 'w'): pass # add basic configuration logging.basicConfig(filename=LOG_FILENAME, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=config.getint('basic_settings', 'file_log_level')) # add a handler for a console output console = logging.StreamHandler() console.setLevel(config.getint('basic_settings', 'console_log_level')) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') console.setFormatter(formatter) logging.getLogger('').addHandler(console) return def synchronize_csv(config, test_mode): """Iterates through csv list of users and synchronizes their messages.""" # initialize loop permanent data caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header')) date_parser = MailDateParser() server = config.get('basic_settings', 'imap_server') tolerance = config.getint('basic_settings', 'tolerance_mins') * 60 skip_shared_folders = config.getboolean('basic_settings', 'skip_shared_folders') # iterate through the users in the csv data user_reader = csv.DictReader(open(CSV_FILENAME, "r"), delimiter=',') for user in user_reader: try: session = MailIterator(server, user['username'], user['password'], skip_shared_folders) except UserWarning as ex: logging.error(ex) continue for mailbox in session: try: box = caching_data.retrieve_cached_mailbox(mailbox[0], mailbox[1], user['username']) mail_ids = session.fetch_messages() new_ids = box.synchronize(mail_ids, tolerance) logging.info("%s new messages out of %s found in %s.", len(new_ids), len(mail_ids), box.name) except UserWarning as ex: logging.error(ex) continue for mid in new_ids: try: fetched_internal_date = session.fetch_internal_date(mid) internal_date = date_parser.extract_internal_date(fetched_internal_date) fetched_correct_date = session.fetch_received_date(mid) correct_date = date_parser.extract_received_date(fetched_correct_date) # check for empty received headers if(correct_date == ""): logging.debug("No received date could be found in message uid: %s - mailbox: %s - user: %s.", mid.decode('iso-8859-1'), box.name, box.owner) box.no_received_field += 1 # correct these messages if required and override received_date from basic date if(config.getboolean('basic_settings', 'fallback_to_date_header')): fetched_correct_date = session.fetch_basic_date(mid) correct_date = date_parser.extract_received_date(fetched_correct_date) if(correct_date == ""): logging.debug("No fallback date header could be found in message uid: %s - mailbox: %s - user: %s.", mid.decode('iso-8859-1'), box.name, box.owner) if(correct_date == ""): # skip synchronization for this message continue else: # preserve only the first received line as fetched if everything is ok fetched_correct_date = fetched_correct_date.split("Received:")[1] except UserWarning as ex: logging.error(ex) continue if(date_parser.compare_dates(correct_date, internal_date, tolerance)): logging.warning("Date conflict found in message uid: %s - mailbox: %s - user: %s.\nInternal date %s is different from extracted date %s from header:\n%s.", mid.decode('iso-8859-1'), box.name, box.owner, internal_date.strftime("%d %b %Y %H:%M:%S"), correct_date.strftime("%d %b %Y %H:%M:%S"), fetched_correct_date) if(not test_mode): try: session.update_message(mid, box.name, correct_date) except UserWarning as ex: logging.error(ex) continue # count total emails for every user and mailbox box.date_conflicts += 1 # if all messages were successfully fixed confirm caching if(not test_mode): box.confirm_change() # final report on date conflicts caching_data.report_conflicts() return if(__name__ == "__main__"): main()