Caching functionality for performance improvement

[imap-fix-internaldate] / fix_imap_internaldate.py

'''
fix_imap_internaldate.py - Fix the INTERNALDATE field on IMAP servers

Copyright (c) 2012 Intra2net AG
Author: Plamen Dimitrov

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

Add '-t' argument when running the module for a test mode.
For a detailed list of each message with a date conflict change
the 'log_level' in the configuration file from '30' to '20'.
'''

import sys
import csv
import logging
import configparser
from date_interpreter import DateInterpreter
from mail_iterator import MailIterator
from caching_data import CachingData

def main():
    """Iterates through csv list of users and their mailboxes"""
    if (len(sys.argv) > 1 and sys.argv[1]=="-t"):
        test_mode = 1
    else:
        test_mode = 0

    config = load_configuration()
    logging.basicConfig(filename='mailscript.log',
                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
                        level=config.getint('basic_settings', 'log_level'))

    date_interp = DateInterpreter()
    cashing_data = CachingData()
    logging.warning(cashing_data)
    user_reader = csv.DictReader(open("userdata.csv", "r"), delimiter=',')

    server = config.get('basic_settings', 'imap_server')
    tolerance = config.getint('basic_settings', 'tolerance')
    total_per_box = {}

    for user in user_reader:
        try:
            session = MailIterator(server, user['username'], user['password'])
        except UserWarning as ex:
            logging.error(ex)
            continue
        for mailbox in session:
            try:
                #special key to ensure better mailbox uniqueness
                mailbox_key = mailbox[0].strip('"') + mailbox[1]
                mail_ids = session.fetch_messages()
                new_ids = cashing_data.sync_cached_mailbox(user['username'], mailbox_key, mail_ids)
                #print(len(new_ids), "new out of", len(mail_ids), "in", mailbox)
            except UserWarning as ex:
                logging.error(ex)
                continue
            for mid in new_ids:
                try:
                    fetched_internal_date = session.fetch_internal_date(mid)
                    internal_date = date_interp.extract_internal_date(fetched_internal_date)
                    fetched_received_date = session.fetch_received_date(mid)
                    received_date = date_interp.extract_received_date(fetched_received_date)
                    if(received_date==""):
                        logging.warning("No received date could be found in message uid: %s - mailbox: %s - user: %s.\n",
                                        mid.decode("utf-8"), mailbox[0], user['username'])
                        continue
                except UserWarning as ex:
                    logging.error(ex)
                    continue
                if(date_interp.compare_dates(received_date, internal_date, tolerance)):
                    #print(received_date, internal_date)
                    if(test_mode==0):
                        try:
                            session.update_message(mid, mailbox[0], received_date)
                        except UserWarning as ex:
                            logging.error(ex)
                            continue
                    else:
                        logging.info("Date conflict found in message uid: %s - mailbox: %s - user: %s.\nInternal date %s is different from received date %s from RECEIVED header:\n%s.",
                                        mid.decode("utf-8"), mailbox[0], user['username'],
                                        internal_date.strftime("%d %b %Y %H:%M:%S"),
                                        received_date.strftime("%d %b %Y %H:%M:%S"),
                                        fetched_received_date[0][1].decode("utf-8").split("Received:")[1])
                    # count total emails for every user and mailbox
                    user_key = user['username']+'|'+mailbox[0].strip('"')
                    total_per_box[user_key] = 1 + total_per_box.get(user_key, 0)
            # if all messages were successfully fixed confirm caching
            cashing_data.commit_cached_mailbox(user['username'], mailbox_key)
        # final report on date conflicts
        total_per_user = 0
        for warning in total_per_box:
            total_per_user += total_per_box[warning]
            logging.warning("Total date conflicts to be corrected in a mailbox %s are %s.",
                         warning.split('|')[1], total_per_box[warning])
        logging.warning("Total date conflicts to be corrected for user %s are %s.\n",
                     user['username'], total_per_user)

def load_configuration():
    """Loads the script configuration from a file or creates such."""
    config = configparser.RawConfigParser()    
    try:
        config.read('confscript.cfg')
    except IOError:
        config.add_section('basic_settings')
        config.set('basic_settings', 'log_level', logging.DEBUG)
        #config.set('Basic settings', 'bool', 'true')
        with open('confscript.cfg', 'w') as configfile:
            config.write(configfile)
    return config

if(__name__ == "__main__"):
    main()