From e2e13005dcb2ad8f969463a6ab8a1af7a048c001 Mon Sep 17 00:00:00 2001 From: Christian Herdtweck Date: Tue, 22 Jan 2019 10:50:10 +0100 Subject: [PATCH] log_read: create/Fix docstrings Including fixes inspired by Plamen's review. --- src/log_read.py | 134 ++++++++++++++++++++++++++++++++++++++---------------- 1 files changed, 94 insertions(+), 40 deletions(-) diff --git a/src/log_read.py b/src/log_read.py index 52b8cec..07f9ac0 100644 --- a/src/log_read.py +++ b/src/log_read.py @@ -18,14 +18,24 @@ # # Copyright (c) 2016-2018 Intra2net AG -""" Iterative reading of log files +""" + +SUMMARY +------------------------------------------------------ +Iterative reading of log files, similar to shell command `tail -f`. + +Copyright: Intra2net AG + + +CONTENTS +------------------------------------------------------ Basic Functionality (class :py:class:`IterativeReader`): Runs stat in a loop to find out whether file size has changed. Then reads the new data and forwards that -.. todo:: Want to also use lsof to find out whether file/pipe/socket was closed, - so can return from read loop +.. todo:: Want to also use lsof to find out whether file/pipe/socket was + closed, so can return from read loop :py:class:`LineReader` takes output of :py:class:`IterativeReader` and returns it line-wise as is normal for log files @@ -34,6 +44,11 @@ it line-wise as is normal for log files like date, time, module name, urgency and message. .. todo:: auto-detect log line layout + + +INTERFACE +------------------------------------------------------ + """ import os @@ -46,34 +61,41 @@ from .type_helpers import is_str_or_byte, is_file_obj class LogReadWarning(UserWarning): - """ warnings issued by classes in this module """ + """Warnings issued by classes in this module.""" pass -def true_func(unused_argument_but_that_is_ok): - """ does nothing, always returns True """ +def true_func(_): + """Replacement for :py:func:`check_is_used`. Returns `True` always.""" return True -def check_is_used(some_file_or_handle): - """ check whether file is being written to +def check_is_used(file_handle): + """ + Check whether file is being written to. + + To be implemented, e.g. using lsof. - to be implemented, e.g. using lsof + If beneficial could also easily supply python file object as arg. + + :param int file_handle: OS-level file descriptor """ - raise NotImplementedError() + raise NotImplementedError(file_handle) +#: counter for unknown sources in :py:func:`create_description` _create_description_unknown_counter = 0 def create_description(file_obj, file_desc): - """ create some description for given file-like object / file descriptor + """ + Create some description for given file-like object / file descriptor. :param file_obj: file-like object :param int file_desc: os-level file descriptor - :returns: string + :returns: Short description for file-like object + :rtype: string """ - global _create_description_unknown_counter try: @@ -96,21 +118,21 @@ _STR_ERR = 'not accepting file name "{0}" since cannot guarantee closing ' \ class IterativeReader(object): - """ reads from a given file - - Uses os.stat(file_obj.fileno()).st_size as measure whether file has changed - or not; Always reads as much data as possible + """ + Read continuously from a given file. - Catches most common exceptions in iteration (not constructor) + Use `os.stat(file_obj.fileno()).st_size` as measure whether file has + changed or not; Always reads as much data as possible. - Does not care about closing files, so does not accept file names + Does not care about closing files, so does not accept file names. This is the base for class :py:class:`LineReader` that just has to - implement a different :py:meth:`prepare_result` method + implement a different :py:meth:`prepare_result` method. """ def __init__(self, sources, descs=None, return_when_done=False): - """ creates a reader; does some basic checks on args + """ + Create a reader; do some basic checks on args. :param sources: iterable over sources. Sources can be opened file objects or read-opened os-level file descriptors. @@ -199,12 +221,20 @@ class IterativeReader(object): .format(file_desc, obj, description)) def n_sources(self): + """Return number of sources given to constructor.""" return len(self.file_objs) def n_active_sources(self): + """Return number of sources we are actually watching.""" return len(self.ignore) - sum(self.ignore) def __iter__(self): + """ + Continue reading from sources, yield results. + + yields result of :py:meth:`prepare_result`, which depends on what sub + class you called this function from. + """ while True: for idx, (obj, file_desc, description, last_size, do_ignore) in \ enumerate(zip(self.file_objs, self.file_descs, @@ -253,47 +283,60 @@ class IterativeReader(object): self.last_sizes[idx] = new_size def prepare_result(self, description, data, idx): - """ from raw new data create some yield-able results + """ + From raw new data create some yield-able results. - to be intended for overwriting in sub-classes + Intended for overwriting in sub-classes. - this function is called from __iter__ for each new data that becomes + This function is called from __iter__ for each new data that becomes available. It has to return some iterable whose entries are yielded from iteration over objects of this class. - It receives the following args: - - the description of the source - - the data itself - - the index of the source The result must be an iterable of objects, which are yielded as-is, so - can have any form + can have any form. This base implementation just returns its input in a list, so new data - is yielded from __iter__ as-is + is yielded from __iter__ as-is. + + Subclass implementations can also yield tuples. + + :param str description: Description of source of lines, one of + :py:data:`self.descriptions` + :param str new_data: Text data read from source + :param idx: Index of data source + :returns: [(description, data)], same as input + :rtype [(str, str)] """ return [(description, data), ] +#: characters to `rstrip()` from end of complete lines LINE_SPLITTERS = '\n\r' class LineReader(IterativeReader): - """ an IterativeReader that returns new data line-wise + """ + An :py:class:`IterativeReader` that returns new data line-wise. - this means buffering partial line data + This means buffering partial line data. """ def __init__(self, *args, **kwargs): - """ forwards all args and kwargs to :py:class:`IterativeReader` """ + """Create an :py:class:`IterativeReader and buffers for sources.""" super(LineReader, self).__init__(*args, **kwargs) self.line_buffers = ['' for _ in range(self.n_sources())] def prepare_result(self, description, new_data, idx): - """ take raw new data and split it into lines + """ + Take raw new data and split it into lines. - if line is not complete, then buffer it + If line is not complete, then buffer it. - returns lines without their newline characters + Args: see super class method :py:meth:`IterativeReader.prepare_result` + :returns: list of 2-tuples `(description, line)` where + `description` is same as arg, and `line` is + without trailing newline characters + :rtype: [(str, str)] """ all_data = self.line_buffers[idx] + new_data self.line_buffers[idx] = '' @@ -313,26 +356,37 @@ class LineReader(IterativeReader): class LogParser(LineReader): - """ takes lines from LineReader and parses their contents + """ + Takes lines from :py:class:`LineReader` and parses their contents. - requires a pattern for log lines, auto-detection is not implemented yet + Requires a pattern for log lines, auto-detection is not implemented yet. Iteration returns re.match result or -- if matching failed -- the original - raw line + raw line. """ def __init__(self, log_file, pattern=None): - """ create a LogParser + """ + Create a LogParser. :param str log_file: name of log file to parse (required!) :param pattern: regexp to split log lines; None (default) to return line as they are + :type pattern: str or None (default) """ super(LogParser, self).__init__(log_file) self.pattern = pattern def prepare_result(self, *args): + """ + Try parsing lines. + + Args: see super class method :py:meth:`IterativeReader.prepare_result` + :returns: either a :py:class:`re.Match` if line matched + :py:data:`self.pattern` or just str if line did not match. + :rtype: :py:class:`re.Match` OR str + """ # let super class split data into lines for _, raw_line in super(LogParser, self).prepare_result(*args): result = re.match(self.pattern, raw_line) -- 1.7.1