Commit | Line | Data |
---|---|---|
c9da760a PD |
1 | ''' |
2 | fix_imap_internaldate.py - Fix the INTERNALDATE field on IMAP servers | |
3 | ||
4 | Copyright (c) 2012 Intra2net AG | |
28d0a914 | 5 | Author: Plamen Dimitrov and Thomas Jarosch |
c9da760a PD |
6 | |
7 | This program is free software: you can redistribute it and/or modify | |
8 | it under the terms of the GNU General Public License as published by | |
9 | the Free Software Foundation, either version 3 of the License, or | |
10 | (at your option) any later version. | |
11 | ||
12 | This program is distributed in the hope that it will be useful, | |
13 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | GNU General Public License for more details. | |
c9da760a PD |
16 | ''' |
17 | ||
d0cfa9d0 | 18 | import sys |
c9da760a | 19 | import csv |
d0cfa9d0 | 20 | import argparse |
25f606a8 | 21 | |
d0cfa9d0 PD |
22 | try: |
23 | import configparser | |
24 | except ImportError: | |
25f606a8 TJ |
25 | import ConfigParser as configparser |
26 | ||
8fe4e3ff | 27 | import logging |
8a9d4c89 | 28 | from mail_date_parser import MailDateParser |
c9da760a | 29 | from mail_iterator import MailIterator |
8301e589 | 30 | from caching_data import CachingData |
c9da760a | 31 | |
6177b21d TJ |
32 | CONFIG_FILENAME = "fix_imap_internaldate.cfg" |
33 | LOG_FILENAME = "fix_imap_internaldate.log" | |
34 | CSV_FILENAME = "userdata.csv" | |
35 | ||
c9da760a | 36 | def main(): |
97bd6bea PD |
37 | """Interprets command arguments and initializes configuration and logger. |
38 | Then begins mail synchronization.""" | |
648f0037 | 39 | |
97bd6bea | 40 | # parse arguments |
648f0037 PD |
41 | parser = argparse.ArgumentParser(description="Fix the INTERNALDATE field on IMAP servers. " |
42 | "Small tool to fix the IMAP internaldate " | |
43 | "in case it's too much off compared to the last date " | |
44 | "stored in the received lines.") | |
45 | parser.add_argument('-u', '--update', dest='test_mode', action='store_false', | |
46 | default=True, help='update all e-mails and exit test mode') | |
c9da760a | 47 | |
8fe4e3ff | 48 | # config and logging setup |
c9da760a | 49 | config = load_configuration() |
8fe4e3ff | 50 | prepare_logger(config) |
97bd6bea | 51 | args = parser.parse_args() |
b3accd58 TJ |
52 | |
53 | # check for update mode in the config | |
54 | if config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing'): | |
55 | args.test_mode = False | |
56 | ||
648f0037 | 57 | if(args.test_mode): |
f9fd9b29 | 58 | logging.info("Testing mode initiated. No message will be modified on the server.") |
648f0037 | 59 | else: |
f9fd9b29 | 60 | logging.info("Update mode initiated. Messages will be modified.") |
3b81023f | 61 | |
97bd6bea | 62 | # proceed to main functionality |
d0cfa9d0 PD |
63 | try: |
64 | synchronize_csv(config, args.test_mode) | |
65 | except KeyboardInterrupt: | |
66 | logging.info("Script was interrupted by the user.") | |
97bd6bea | 67 | |
28d8aa17 | 68 | logging.info("All done. Exiting.") |
97bd6bea PD |
69 | return |
70 | ||
71 | def load_configuration(): | |
72 | """Loads the script configuration from a file or creates such.""" | |
c4974933 | 73 | config = configparser.RawConfigParser() |
6177b21d | 74 | success = config.read(CONFIG_FILENAME) |
c4974933 TJ |
75 | |
76 | # if no file is found create a default one | |
97bd6bea | 77 | if(len(success)==0): |
95467f63 PD |
78 | if(not config.has_section('basic_settings')): |
79 | config.add_section('basic_settings') | |
97bd6bea PD |
80 | config.set('basic_settings', 'file_log_level', logging.INFO) |
81 | config.set('basic_settings', 'console_log_level', logging.INFO) | |
82 | config.set('basic_settings', 'imap_server', 'imap.company.com') | |
a936a06b | 83 | config.set('basic_settings', 'tolerance_mins', 30) |
c4974933 TJ |
84 | config.set('basic_settings', 'skip_shared_folders', "True") |
85 | config.set('basic_settings', 'fallback_to_date_header', "False") | |
b3accd58 | 86 | config.set('basic_settings', 'update_mode_i_know_what_i_am_doing', "False") |
6177b21d | 87 | with open(CONFIG_FILENAME, 'w') as configfile: |
97bd6bea | 88 | config.write(configfile) |
ef29ccfb | 89 | configfile.write("# 0 NOTSET, 10 DEBUG, 20 INFO, 30 WARNING, 40 ERROR, 50 CRITICAL\n") |
c4974933 TJ |
90 | print("Created initial config %s" % CONFIG_FILENAME) |
91 | ||
92 | try: | |
93 | config.get('basic_settings', 'file_log_level') | |
94 | config.get('basic_settings', 'console_log_level') | |
95 | config.get('basic_settings', 'imap_server') | |
96 | config.getint('basic_settings', 'tolerance_mins') | |
97 | config.getboolean('basic_settings', 'skip_shared_folders') | |
98 | config.getboolean('basic_settings', 'fallback_to_date_header') | |
b3accd58 | 99 | config.getboolean('basic_settings', 'update_mode_i_know_what_i_am_doing') |
c4974933 TJ |
100 | except (configparser.NoSectionError, configparser.NoOptionError, ValueError) as ex: |
101 | print("Could not read config file '%s': %s." % (CONFIG_FILENAME, ex)) | |
102 | print("Please change or remove the config file.") | |
103 | sys.exit() | |
ef29ccfb | 104 | |
97bd6bea PD |
105 | return config |
106 | ||
107 | def prepare_logger(config): | |
108 | """Sets up the logging functionality""" | |
6177b21d | 109 | |
97bd6bea | 110 | # reset the log |
6177b21d | 111 | with open(LOG_FILENAME, 'w'): |
97bd6bea | 112 | pass |
6177b21d | 113 | |
97bd6bea | 114 | # add basic configuration |
6177b21d | 115 | logging.basicConfig(filename=LOG_FILENAME, |
97bd6bea PD |
116 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', |
117 | level=config.getint('basic_settings', 'file_log_level')) | |
6177b21d | 118 | |
97bd6bea PD |
119 | # add a handler for a console output |
120 | console = logging.StreamHandler() | |
121 | console.setLevel(config.getint('basic_settings', 'console_log_level')) | |
122 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') | |
123 | console.setFormatter(formatter) | |
124 | logging.getLogger('').addHandler(console) | |
125 | return | |
126 | ||
127 | def synchronize_csv(config, test_mode): | |
128 | """Iterates through csv list of users and synchronizes their messages.""" | |
129 | ||
130 | # initialize loop permanent data | |
db3f09a6 | 131 | caching_data = CachingData(config.getboolean('basic_settings', 'fallback_to_date_header')) |
97bd6bea | 132 | date_parser = MailDateParser() |
c9da760a | 133 | server = config.get('basic_settings', 'imap_server') |
a936a06b | 134 | tolerance = config.getint('basic_settings', 'tolerance_mins') * 60 |
c4974933 | 135 | skip_shared_folders = config.getboolean('basic_settings', 'skip_shared_folders') |
c9da760a | 136 | |
97bd6bea | 137 | # iterate through the users in the csv data |
6177b21d | 138 | user_reader = csv.DictReader(open(CSV_FILENAME, "r"), delimiter=',') |
c9da760a PD |
139 | for user in user_reader: |
140 | try: | |
c4974933 | 141 | session = MailIterator(server, user['username'], user['password'], skip_shared_folders) |
c9da760a PD |
142 | except UserWarning as ex: |
143 | logging.error(ex) | |
144 | continue | |
145 | for mailbox in session: | |
146 | try: | |
7a1d4c35 | 147 | box = caching_data.retrieve_cached_mailbox(mailbox[0], mailbox[1], user['username']) |
c9da760a | 148 | mail_ids = session.fetch_messages() |
3b81023f | 149 | new_ids = box.synchronize(mail_ids, tolerance) |
6f2bc406 | 150 | logging.info("%s new messages out of %s found in %s.", len(new_ids), len(mail_ids), box.name) |
c9da760a PD |
151 | except UserWarning as ex: |
152 | logging.error(ex) | |
153 | continue | |
8301e589 | 154 | for mid in new_ids: |
c9da760a PD |
155 | try: |
156 | fetched_internal_date = session.fetch_internal_date(mid) | |
8a9d4c89 | 157 | internal_date = date_parser.extract_internal_date(fetched_internal_date) |
87cde111 PD |
158 | fetched_correct_date = session.fetch_received_date(mid) |
159 | correct_date = date_parser.extract_received_date(fetched_correct_date) | |
160 | # check for empty received headers | |
161 | if(correct_date == ""): | |
d5ccfbec | 162 | logging.debug("No received date could be found in message uid: %s - mailbox: %s - user: %s.", |
8a9d4c89 | 163 | mid.decode('iso-8859-1'), box.name, box.owner) |
7a1d4c35 | 164 | box.no_received_field += 1 |
87cde111 | 165 | # correct these messages if required and override received_date from basic date |
a05fef0a | 166 | if(config.getboolean('basic_settings', 'fallback_to_date_header')): |
87cde111 PD |
167 | fetched_correct_date = session.fetch_basic_date(mid) |
168 | correct_date = date_parser.extract_received_date(fetched_correct_date) | |
031ddbdf TJ |
169 | if(correct_date == ""): |
170 | logging.debug("No fallback date header could be found in message uid: %s - mailbox: %s - user: %s.", | |
171 | mid.decode('iso-8859-1'), box.name, box.owner) | |
172 | ||
173 | if(correct_date == ""): | |
87cde111 PD |
174 | # skip synchronization for this message |
175 | continue | |
176 | else: | |
177 | # preserve only the first received line as fetched if everything is ok | |
178 | fetched_correct_date = fetched_correct_date.split("Received:")[1] | |
c9da760a PD |
179 | except UserWarning as ex: |
180 | logging.error(ex) | |
181 | continue | |
87cde111 PD |
182 | if(date_parser.compare_dates(correct_date, internal_date, tolerance)): |
183 | logging.warning("Date conflict found in message uid: %s - mailbox: %s - user: %s.\nInternal date %s is different from extracted date %s from header:\n%s.", | |
36f4d196 TJ |
184 | mid.decode('iso-8859-1'), box.name, box.owner, |
185 | internal_date.strftime("%d %b %Y %H:%M:%S"), | |
87cde111 PD |
186 | correct_date.strftime("%d %b %Y %H:%M:%S"), |
187 | fetched_correct_date) | |
5a4bc5d8 | 188 | if(not test_mode): |
c9da760a | 189 | try: |
87cde111 | 190 | session.update_message(mid, box.name, correct_date) |
c9da760a PD |
191 | except UserWarning as ex: |
192 | logging.error(ex) | |
193 | continue | |
36f4d196 | 194 | |
8301e589 | 195 | # count total emails for every user and mailbox |
7a1d4c35 | 196 | box.date_conflicts += 1 |
36f4d196 | 197 | |
8301e589 | 198 | # if all messages were successfully fixed confirm caching |
97bd6bea | 199 | if(not test_mode): |
7a1d4c35 | 200 | box.confirm_change() |
6177b21d | 201 | |
08ba5736 PD |
202 | # final report on date conflicts |
203 | caching_data.report_conflicts() | |
8fe4e3ff PD |
204 | return |
205 | ||
c9da760a PD |
206 | if(__name__ == "__main__"): |
207 | main() |