Style improvements and cache from settings validation added

[imap-fix-internaldate] / src / fix_imap_internaldate.py

'''
fix_imap_internaldate.py - Fix the INTERNALDATE field on IMAP servers

Copyright (c) 2012 Intra2net AG
Author: Plamen Dimitrov

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
'''

import sys
import csv
import argparse
# python version handling
try:
    import configparser
except ImportError:
    print("This module needs python version 3 or later.")
    sys.exit()
import logging
from mail_date_parser import MailDateParser
from mail_iterator import MailIterator
from caching_data import CachingData

CONFIG_FILENAME = "fix_imap_internaldate.cfg"
LOG_FILENAME = "fix_imap_internaldate.log"
CSV_FILENAME = "userdata.csv"

def main():
    """Interprets command arguments and initializes configuration and logger.
        Then begins mail synchronization."""

    # parse arguments
    parser = argparse.ArgumentParser(description="Fix the INTERNALDATE field on IMAP servers. "
                                                "Small tool to fix the IMAP internaldate "
                                                "in case it's too much off compared to the last date "
                                                "stored in the received lines.")
    parser.add_argument('-u', '--update', dest='test_mode', action='store_false',
                        default=True, help='update all e-mails and exit test mode')

    # config and logging setup
    config = load_configuration()
    prepare_logger(config)
    args = parser.parse_args()
    if(args.test_mode):
        logging.info("Testing mode initiated. No message will be modified on the server.")
    else:
        logging.info("Update mode initiated. Messages will be modified.")

    # proceed to main functionality
    try:
        synchronize_csv(config, args.test_mode)
    except KeyboardInterrupt:
        logging.info("Script was interrupted by the user.")

    logging.info("All done. Exiting.")
    return

def load_configuration():
    """Loads the script configuration from a file or creates such."""
    config = configparser.RawConfigParser()    
    success = config.read(CONFIG_FILENAME)
    try:
        config.get('basic_settings', 'file_log_level')
        config.get('basic_settings', 'console_log_level')
        config.get('basic_settings', 'imap_server')
        config.getint('basic_settings', 'tolerance_mins')
        config.getboolean('basic_settings', 'skip_shared_folders')
        config.getboolean('basic_settings', 'fallback_to_date_header')
    except configparser.NoSectionError:
        success = []
    except configparser.NoOptionError:
        success = []
    except ValueError:
        success = []

    # if corrupted settings save file and load default
    if(len(success)==0):
        if(not config.has_section('basic_settings')):
            config.add_section('basic_settings')
        config.set('basic_settings', 'file_log_level', logging.INFO)
        config.set('basic_settings', 'console_log_level', logging.INFO)
        config.set('basic_settings', 'imap_server', 'imap.company.com')
        config.set('basic_settings', 'tolerance_mins', 30)
        config.set('basic_settings', 'skip_shared_folders', True)
        config.set('basic_settings', 'fallback_to_date_header', False)
        with open(CONFIG_FILENAME, 'w') as configfile:
            config.write(configfile)
            configfile.write("# 0 NOTSET, 10 DEBUG, 20 INFO, 30 WARNING, 40 ERROR, 50 CRITICAL\n")

    return config

def prepare_logger(config):
    """Sets up the logging functionality"""

    # reset the log
    with open(LOG_FILENAME, 'w'):
        pass

    # add basic configuration
    logging.basicConfig(filename=LOG_FILENAME,
                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                        level=config.getint('basic_settings', 'file_log_level'))

    # add a handler for a console output
    console = logging.StreamHandler()
    console.setLevel(config.getint('basic_settings', 'console_log_level'))
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
    return

def synchronize_csv(config, test_mode):
    """Iterates through csv list of users and synchronizes their messages."""

    # initialize loop permanent data
    caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header'))  
    date_parser = MailDateParser()
    server = config.get('basic_settings', 'imap_server')
    tolerance = config.getint('basic_settings', 'tolerance_mins') * 60

    # iterate through the users in the csv data
    user_reader = csv.DictReader(open(CSV_FILENAME, "r"), delimiter=',')
    for user in user_reader:
        try:
            session = MailIterator(server, user['username'], user['password'],
                                   config.getboolean('basic_settings', 'skip_shared_folders'))
        except UserWarning as ex:
            logging.error(ex)
            continue
        for mailbox in session:
            try:
                box = caching_data.retrieve_cached_mailbox(mailbox[0], mailbox[1], user['username'])
                mail_ids = session.fetch_messages()
                new_ids = box.synchronize(mail_ids, tolerance)
                logging.info("%s new messages out of %s found in %s.", len(new_ids), len(mail_ids), box.name)
            except UserWarning as ex:
                logging.error(ex)
                continue
            for mid in new_ids:
                try:
                    fetched_internal_date = session.fetch_internal_date(mid)
                    internal_date = date_parser.extract_internal_date(fetched_internal_date)
                    fetched_correct_date = session.fetch_received_date(mid)
                    correct_date = date_parser.extract_received_date(fetched_correct_date)
                    # check for empty received headers
                    if(correct_date == ""):
                        logging.debug("No received date could be found in message uid: %s - mailbox: %s - user: %s.",
                                        mid.decode('iso-8859-1'), box.name, box.owner)
                        box.no_received_field += 1
                        # correct these messages if required and override received_date from basic date
                        if(config.getboolean('basic_settings', 'fallback_to_date_header')):
                            fetched_correct_date = session.fetch_basic_date(mid)
                            correct_date = date_parser.extract_received_date(fetched_correct_date)
                            if(correct_date == ""):
                                logging.debug("No fallback date header could be found in message uid: %s - mailbox: %s - user: %s.",
                                                mid.decode('iso-8859-1'), box.name, box.owner)

                        if(correct_date == ""):
                            # skip synchronization for this message
                            continue
                    else:
                        # preserve only the first received line as fetched if everything is ok
                        fetched_correct_date = fetched_correct_date.split("Received:")[1]
                except UserWarning as ex:
                    logging.error(ex)
                    continue
                if(date_parser.compare_dates(correct_date, internal_date, tolerance)):
                    logging.warning("Date conflict found in message uid: %s - mailbox: %s - user: %s.\nInternal date %s is different from extracted date %s from header:\n%s.",
                                    mid.decode('iso-8859-1'), box.name, box.owner,
                                    internal_date.strftime("%d %b %Y %H:%M:%S"),
                                    correct_date.strftime("%d %b %Y %H:%M:%S"),
                                    fetched_correct_date)
                    if(not test_mode):
                        try:
                            session.update_message(mid, box.name, correct_date)
                        except UserWarning as ex:
                            logging.error(ex)
                            continue

                    # count total emails for every user and mailbox
                    box.date_conflicts += 1

            # if all messages were successfully fixed confirm caching
            if(not test_mode):
                box.confirm_change()

    # final report on date conflicts
    caching_data.report_conflicts()
    return

if(__name__ == "__main__"):
    main()