From: Christian Herdtweck Date: Mon, 30 Nov 2015 11:59:31 +0000 (+0100) Subject: fixed and added to I2nLogger, is now in a useable state X-Git-Tag: v1.2~92 X-Git-Url: http://developer.intra2net.com/git/?a=commitdiff_plain;h=2e6f474f1623556c6597b53c94c2df445d34eceb;p=pyi2ncommon fixed and added to I2nLogger, is now in a useable state --- diff --git a/log_helpers.py b/log_helpers.py index 605febb..a2830f6 100644 --- a/log_helpers.py +++ b/log_helpers.py @@ -16,16 +16,27 @@ # This exception does not invalidate any other reasons why a work based # on this file might be covered by the GNU General Public License. -""" Helpers for logging +""" Helpers for logging; featuring: ShortLevelFormatter: provide a 4-char-sized field "shortlevel" for message urgency (dbug/info/warn/err /crit) I2nLogger: logger that provides a notice(), allows omission for str.format + and is quite convenient in general + +get_logger: factor for creating I2nLoggers Further ideas: :: * allow milliseconds in dateformat field (see test_short_level_format) * create own basicConfig-like function that uses our classes as default + --> started using I2nLogger constructor and get_logger +* try to find out module that calls I2nLogger constructor to provide a good + default value for name (finding out the current module is a pain in the a..) + +..todo:: create unittests from test_* functions at bottom +..todo:: think about how to allow different levels per handler +..todo:: do not limit logs by line numbers but by disc size? Warn when at 50%, + 75%, 90%, 99% of limit? .. codeauthor:: Christian Herdtweck, christian.herdtweck@intra2net.com """ @@ -33,6 +44,9 @@ Further ideas: :: import logging from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL, NOTSET from math import log10, floor +import sys + +from type_helpers import isstr #: log level half-way between INFO and WARNING NOTICE = (INFO + WARNING)/2 @@ -54,6 +68,10 @@ MIN_LEVEL = NOTSET #: max level allowed in I2nLogger MAX_LEVEL = CRITICAL +#: constant for I2nLogger streams argument: stream to stdout +STDOUT = sys.stdout + + class ShortLevelFormatter(logging.Formatter): """ Formatter for logging handlers that allows use of format field "shortlevel" @@ -66,7 +84,10 @@ class ShortLevelFormatter(logging.Formatter): All other functionality (like other format fields) is inherited from base class Formatter - Usage example:: + Most easily used indirectly through :py:class:`I2nLogger` + (uses this by default) + + Explicit usage example:: logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -119,11 +140,29 @@ class ShortLevelFormatter(logging.Formatter): self._shortlevel_dict[levelno] = shortlevel_str +#: memory for all I2nLogger instances in current session to avoid overwrite +#: used in :py:func:`get_logger` and :py:class:`I2nLogger` constructor +_i2n_loggers = {} + + +def get_logger(name, *args, **kwargs): + """ factor for :py:class:`I2nLogger`: create new or return existing object + + all args as in :py:class:`I2nLogger` constructor; will be ignored if logger + with given name exists already! + """ + + if name in _i2n_loggers: + return _i2n_loggers[name] # ignore all args + else: + return I2nLogger(name, *args, **kwargs) + + class I2nLogger: """ a more convenient logger Features:: - * can be used as follows:: + * can be used without ".format()" as follows:: logger.info('the result is {0}', result) # more convenient than... logger.info('the result is {0}'.format(result)) # ... the original @@ -134,33 +173,92 @@ class I2nLogger: * Simplifies setting of logger's logging level using :py:meth:`Logger.set_level`. The level is kept in sync between logger and handlers and can be queried using :py:meth:`get_level` + (does not work accross logger hierarchies!) * can specify name and level and [date]fmt all in constructor - - ..note:: Creating multiple instances with the same name is possible but - will internally use the same logger. Each instance overwrites at - construction time the previous instances' handlers and levels! + * can limit the number of lines this logger will produce to prevent filling + hard drive with log file + (assumes one line per call to log/debug/info/notice/..., + only counts calls with priority above this logger's level) + * provides shorter method names: dbg, note, warn, err + + ..note:: Creating multiple instances with the same name is not allowed and + will result in an error. Use :py:func:`get_logger` instead of this + constructor to avoid such situtations + + ..note:: Do not change or use the underlying logger or functionality here + (in particular line counting, get_level) is no longer reliable! """ def __init__(self, name, level=INFO, fmt=DEFAULT_SHORT_LEVEL_FORMAT, - datefmt=DEFAULT_SHORT_LEVEL_DATE_FORMAT): + datefmt=DEFAULT_SHORT_LEVEL_DATE_FORMAT, + streams=STDOUT, files=None, max_lines=None): """ creates a I2nLogger; forwards args to logging.getLogger - ..note:: You should use a different name in each constructor call! + + :param str name: name of this logger, best practice is module name + :param int level: best use one of the constants DEBUG, INFO NOTICE, + WARNING, ERROR + :param str fmt: format of log messages, see + :py:class:`ShortLevelFormatter` for more help + :param str datefmt: format of date added to log messages, see + :py:class:`ShortLevelFormatter` for more help + :param streams: list/tuple of or a single stream to log to, default is + STDOUT (=sys.stdout) + :param files: list/tuple or single file name to log to + :param max_lines: number > 0 to limit number of output calls to that + number; give None (default) to no limit + :raise: ValueError if an I2nLogger with the same name exists already """ - self._level = min(MAX_LEVEL, max(MIN_LEVEL, level)) + + # check if an instance with the same name has been created earlier + # to prevent conflicts + global _i2n_loggers + if name in _i2n_loggers: + raise ValueError("An I2nLogger with that exact name ('{0}') exists" + " already -- use get_logger instead!" + .format(name)) + self._log = logging.getLogger(name) - self._log.setLevel(self.level) + self._level = min(MAX_LEVEL, max(MIN_LEVEL, level)) + self._log.setLevel(self._level) # remove handlers (sometimes there are mutliple by default) for handler in self._log.handlers: self._log.removeHandler(handler) - # create new handler and formatter - formatter = ShortLevelFormatter(fmt=fmt, datefmt=datefmt) - formatter.add_level(notice, 'note') - stdout_handler = logging.StreamHandler(formatter) - stdout_handler.setLevel(self.level) - self._log.addHandler(stdout_handler) + # create new handlers and formatter + if streams is None: + stream = [] + elif not isinstance(streams, (list, tuple)): + streams = (streams, ) + for stream in streams: + formatter = ShortLevelFormatter(fmt=fmt, datefmt=datefmt) + formatter.add_level(NOTICE, 'note') + new_handler = logging.StreamHandler(stream) + new_handler.setFormatter(formatter) + new_handler.setLevel(self._level) + self._log.addHandler(new_handler) + + if files is None: + files = [] + elif not isinstance(files, (list, tuple)): + files = (files, ) + for file_name in files: + formatter = ShortLevelFormatter(fmt=fmt, datefmt=datefmt) + formatter.add_level(NOTICE, 'note') + new_handler = logging.FileHandler(file_name) + new_handler.setFormatter(formatter) + new_handler.setLevel(self._level) + self._log.addHandler(new_handler) + + # remember max_lines + self.set_max_lines(max_lines) + + # remember that this logger is a I2nLogger + _i2n_loggers[name] = self + + def dbg(self, message, *args, **kwargs): + self.log(DEBUG, message, *args, **kwargs) def debug(self, message, *args, **kwargs): self.log(DEBUG, message, *args, **kwargs) @@ -168,12 +266,21 @@ class I2nLogger: def info(self, message, *args, **kwargs): self.log(INFO, message, *args, **kwargs) + def note(self, message, *args, **kwargs): + self.log(NOTICE, message, *args, **kwargs) + def notice(self, message, *args, **kwargs): self.log(NOTICE, message, *args, **kwargs) + def warn(self, message, *args, **kwargs): + self.log(WARNING, message, *args, **kwargs) + def warning(self, message, *args, **kwargs): self.log(WARNING, message, *args, **kwargs) + def err(self, message, *args, **kwargs): + self.log(ERROR, message, *args, **kwargs) + def error(self, message, *args, **kwargs): self.log(ERROR, message, *args, **kwargs) @@ -182,7 +289,28 @@ class I2nLogger: def log(self, level, message, *args, **kwargs): if level >= self._level: - self._log.log(level, message.format(*args), **kwargs) + if self._line_counter == self._max_lines: + self._log.log(ERROR, + 'reached max number of output lines ({0}) ' + '-- will not log anything any more!' + .format(self._line_counter)) + self._line_counter += 1 + elif self._line_counter > self._max_lines: + return + else: + self._log.log(level, message.format(*args), **kwargs) + self._line_counter += 1 + + def log_count_if_interesting(self, count, level=INFO, counter_name=None): + """ Log value of a counter in gradually coarser intervals + + see :py:func:`is_interesting_count` for definition of "interesting" + """ + if is_interesting_count(count): + if counter_name: + self.log(level, '{0} counter is at {1}', counter_name, count) + else: + self.log(level, 'Counter is at {0}', count) def get_level(self): """ return int level of this logger """ @@ -202,11 +330,37 @@ class I2nLogger: if isstr(new_level): self._level = LEVEL_DICT[new_level.lower()] else: - self._level = min(MAX_LEVEL, max(MIN_LEVEL, level)) + self._level = min(MAX_LEVEL, max(MIN_LEVEL, new_level)) self._log.setLevel(self._level) for handler in self._log.handlers: handler.setLevel(self._level) + def set_max_lines(self, max_lines): + """ limit number of lines this produces; give None to remove limit + + resets the line counter + """ + if max_lines > 0: + self._max_lines = max_lines + elif max_lines < 0 or (not max_lines): + self._max_lines = None + else: + raise ValueError('unexpected value for max_lines: {0}!' + .format(max_lines)) + self._line_counter = 0 + + def get_max_lines(self): + """ return current value for line limit """ + return self._max_lines + + def exceeded_max_lines(self): + """ return True if nothing will be logged because max_lines was reached + """ + if self._max_lines: + return self._line_counter >= self._max_lines + else: + return False + def n_digits(number): """ returns the number of digits a number has in decimal format @@ -238,10 +392,12 @@ def is_interesting_count(counter): log('reached iteration {0}'.format(counter)) counter += 1 + Or implicitly using I2nLogger::log_count_if_interesting(counter) + :returns: True for a few values of counter, False most of the time """ - return float(counter) / 10.**(n_digits(counter)-1.) in (1.,2.,3.,6.) + return float(counter) / 10.**(n_digits(counter)-1.) in (1., 2., 3., 6.) def test_short_level_format(): @@ -285,5 +441,22 @@ def test_short_level_format(): logger.info('done testing') +def test_get_logger(): + log = get_logger('logger_test') + log2 = get_logger('logger_test') + print(log == log2) + + +def test_line_counter(): + log = get_logger('logger_test', max_lines=10) + for idx in range(20): + for _ in range(20): + log.debug('should not show nor count') + print('calling log for idx {0}'.format(idx)) + log.info('logging with idx {0}', idx) + log.log_count_if_interesting(idx) + if __name__ == '__main__': - test_short_level_format() + #test_short_level_format() + #test_get_logger() + test_line_counter()