#
# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
-""" Iterative reading of log files
+"""
+
+SUMMARY
+------------------------------------------------------
+Iterative reading of log files, similar to shell command `tail -f`.
+
+Copyright: Intra2net AG
+
+
+CONTENTS
+------------------------------------------------------
Basic Functionality (class :py:class:`IterativeReader`):
Runs stat in a loop to find out whether file size has changed. Then reads the
new data and forwards that
-.. todo:: Want to also use lsof to find out whether file/pipe/socket was closed,
- so can return from read loop
+.. todo:: Want to also use lsof to find out whether file/pipe/socket was
+ closed, so can return from read loop
:py:class:`LineReader` takes output of :py:class:`IterativeReader` and returns
it line-wise as is normal for log files
like date, time, module name, urgency and message.
.. todo:: auto-detect log line layout
+
+
+INTERFACE
+------------------------------------------------------
+
"""
import os
class LogReadWarning(UserWarning):
- """ warnings issued by classes in this module """
+ """Warnings issued by classes in this module."""
pass
-def true_func(unused_argument_but_that_is_ok):
- """ does nothing, always returns True """
+def true_func(_):
+ """Replacement for :py:func:`check_is_used`. Returns `True` always."""
return True
-def check_is_used(some_file_or_handle):
- """ check whether file is being written to
+def check_is_used(file_handle):
+ """
+ Check whether file is being written to.
+
+ To be implemented, e.g. using lsof.
- to be implemented, e.g. using lsof
+ If beneficial could also easily supply python file object as arg.
+
+ :param int file_handle: OS-level file descriptor
"""
- raise NotImplementedError()
+ raise NotImplementedError(file_handle)
+#: counter for unknown sources in :py:func:`create_description`
_create_description_unknown_counter = 0
def create_description(file_obj, file_desc):
- """ create some description for given file-like object / file descriptor
+ """
+ Create some description for given file-like object / file descriptor.
:param file_obj: file-like object
:param int file_desc: os-level file descriptor
- :returns: string
+ :returns: Short description for file-like object
+ :rtype: string
"""
-
global _create_description_unknown_counter
try:
class IterativeReader(object):
- """ reads from a given file
-
- Uses os.stat(file_obj.fileno()).st_size as measure whether file has changed
- or not; Always reads as much data as possible
+ """
+ Read continuously from a given file.
- Catches most common exceptions in iteration (not constructor)
+ Use `os.stat(file_obj.fileno()).st_size` as measure whether file has
+ changed or not; Always reads as much data as possible.
- Does not care about closing files, so does not accept file names
+ Does not care about closing files, so does not accept file names.
This is the base for class :py:class:`LineReader` that just has to
- implement a different :py:meth:`prepare_result` method
+ implement a different :py:meth:`prepare_result` method.
"""
def __init__(self, sources, descs=None, return_when_done=False):
- """ creates a reader; does some basic checks on args
+ """
+ Create a reader; do some basic checks on args.
:param sources: iterable over sources. Sources can be opened file
objects or read-opened os-level file descriptors.
.format(file_desc, obj, description))
def n_sources(self):
+ """Return number of sources given to constructor."""
return len(self.file_objs)
def n_active_sources(self):
+ """Return number of sources we are actually watching."""
return len(self.ignore) - sum(self.ignore)
def __iter__(self):
+ """
+ Continue reading from sources, yield results.
+
+ yields result of :py:meth:`prepare_result`, which depends on what sub
+ class you called this function from.
+ """
while True:
for idx, (obj, file_desc, description, last_size, do_ignore) in \
enumerate(zip(self.file_objs, self.file_descs,
self.last_sizes[idx] = new_size
def prepare_result(self, description, data, idx):
- """ from raw new data create some yield-able results
+ """
+ From raw new data create some yield-able results.
- to be intended for overwriting in sub-classes
+ Intended for overwriting in sub-classes.
- this function is called from __iter__ for each new data that becomes
+ This function is called from __iter__ for each new data that becomes
available. It has to return some iterable whose entries are yielded
from iteration over objects of this class.
- It receives the following args:
- - the description of the source
- - the data itself
- - the index of the source
The result must be an iterable of objects, which are yielded as-is, so
- can have any form
+ can have any form.
This base implementation just returns its input in a list, so new data
- is yielded from __iter__ as-is
+ is yielded from __iter__ as-is.
+
+ Subclass implementations can also yield tuples.
+
+ :param str description: Description of source of lines, one of
+ :py:data:`self.descriptions`
+ :param str new_data: Text data read from source
+ :param idx: Index of data source
+ :returns: [(description, data)], same as input
+ :rtype [(str, str)]
"""
return [(description, data), ]
+#: characters to `rstrip()` from end of complete lines
LINE_SPLITTERS = '\n\r'
class LineReader(IterativeReader):
- """ an IterativeReader that returns new data line-wise
+ """
+ An :py:class:`IterativeReader` that returns new data line-wise.
- this means buffering partial line data
+ This means buffering partial line data.
"""
def __init__(self, *args, **kwargs):
- """ forwards all args and kwargs to :py:class:`IterativeReader` """
+ """Create an :py:class:`IterativeReader and buffers for sources."""
super(LineReader, self).__init__(*args, **kwargs)
self.line_buffers = ['' for _ in range(self.n_sources())]
def prepare_result(self, description, new_data, idx):
- """ take raw new data and split it into lines
+ """
+ Take raw new data and split it into lines.
- if line is not complete, then buffer it
+ If line is not complete, then buffer it.
- returns lines without their newline characters
+ Args: see super class method :py:meth:`IterativeReader.prepare_result`
+ :returns: list of 2-tuples `(description, line)` where
+ `description` is same as arg, and `line` is
+ without trailing newline characters
+ :rtype: [(str, str)]
"""
all_data = self.line_buffers[idx] + new_data
self.line_buffers[idx] = ''
class LogParser(LineReader):
- """ takes lines from LineReader and parses their contents
+ """
+ Takes lines from :py:class:`LineReader` and parses their contents.
- requires a pattern for log lines, auto-detection is not implemented yet
+ Requires a pattern for log lines, auto-detection is not implemented yet.
Iteration returns re.match result or -- if matching failed -- the original
- raw line
+ raw line.
"""
def __init__(self, log_file, pattern=None):
- """ create a LogParser
+ """
+ Create a LogParser.
:param str log_file: name of log file to parse (required!)
:param pattern: regexp to split log lines; None (default) to return
line as they are
+ :type pattern: str or None (default)
"""
super(LogParser, self).__init__(log_file)
self.pattern = pattern
def prepare_result(self, *args):
+ """
+ Try parsing lines.
+
+ Args: see super class method :py:meth:`IterativeReader.prepare_result`
+ :returns: either a :py:class:`re.Match` if line matched
+ :py:data:`self.pattern` or just str if line did not match.
+ :rtype: :py:class:`re.Match` OR str
+ """
# let super class split data into lines
for _, raw_line in super(LogParser, self).prepare_result(*args):
result = re.match(self.pattern, raw_line)
git://developer.intra2net.com / pyi2ncommon / commitdiff