[pyi2ncommon] / src / simple_cnf.py

# The software in this package is distributed under the GNU General
# Public License version 2 (with a special exception described below).
#
# A copy of GNU General Public License (GPL) is included in this distribution,
# in the file COPYING.GPL.
#
# As a special exception, if other files instantiate templates or use macros
# or inline functions from this file, or you compile this file and link it
# with other works to produce a work based on this file, this file
# does not by itself cause the resulting work to be covered
# by the GNU General Public License.
#
# However the source code for this file must still be made available
# in accordance with section (3) of the GNU General Public License.
#
# This exception does not invalidate any other reasons why a work based
# on this file might be covered by the GNU General Public License.
#
# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>

"""

SUMMARY
------------------------------------------------------
Read / write / merge guest cnf var sets, even on host.

Copyright: Intra2net AG


CONTENTS
------------------------------------------------------

This module can be viewed as a convenience-wrapper around module
:py:mod:`cnfvar`. It uses many of its function but provides some higher-level
interfaces, most of all class :py:class:`SimpleCnf`. It is completely
independent of the :py:mod:`cnfline` package and its included subclasses
(modules in `shared.cnfline`, starting with ``build_`` and ``configure_``).

Class :py:class:`SimpleCnf` represents a hierarchical set of conf vars and
provides functions to deal with that hierarchy. Under the hood, all functions
here (and probably also in :py:mod:`cnfvar`) work with conf vars represented as
dictionaries and lists thereof. Conf var dicts have keys `number`, `varname`,
`instance`, `data`, `comment` and possibly `parent` and/or `children`.
`varname` is a regular lower-case string, `data` is a (utf8) string, `comment`
is usually None, `number`, `parent` and `instance` are int. If a conf var has
children, then this is a list of conf var dicts. `parent` is only present if a
conf-var is a child. Several conf vars, if not wrapped in a
:py:class:`SimpleCnf`, appear either as simple list of conf var dicts or as a
dict with a single key `cnf` whose value is a list of conf var dicts. (Function
:py:func:`get_cnf` returns the former given the latter).

.. todo:: Exceptions Invalid[Json]Cnf are used inconsistently (e.g. check type
          of function arguments `-->` should be ValueError) and difference
          between them is unclear. Also name differs only in case from
          :py:class:`cnfvar.InvalidCNF`

INTERFACE
------------------------------------------------------
"""

import os
import json
import tempfile
import time
import logging
log = logging.getLogger('pyi2ncommon.simple_cnf')

from . import arnied_wrapper
from . import cnfvar
from . import sysmisc

###############################################################################
#                                  constants
###############################################################################

#: timeout for copying temporary config files to VM objects (seconds)
COPY_FILES_TIMEOUT = 15

#: additional location of configuration files
ADD_CNFFILE_PATH = "/tmp/configs"


###############################################################################
# EXCEPTIONS
###############################################################################


class InvalidCnf(Exception):
    """Exception that indicates a general problem with conf var processing."""

    def __init__(self, m):
        """Create an invalid config exception."""
        msg = "Invalid CNF_VAR: %s" % m
        super(InvalidCnf, self).__init__(msg)
        self.msg = msg
        self.pfx = "[CNF]"

    def __str__(self):
        """Get a string version of the exception message."""
        return "%s %s" % (self.pfx, self.msg)


class InvalidJsonCnf(InvalidCnf):
    """Exception that indicates a general problem with conf var processing."""

    def __init__(self, m):
        """Create an invalid JSON config exception."""
        super(InvalidJsonCnf, self).__init__(m)
        self.pfx = "[CNF:JSON]"


###############################################################################
#                             auxiliary functions
###############################################################################


def get_cnf(cnf):
    """
    "Convert" a config dict to a list of conf var dicts.

    This just removes the top-level 'cnf' key and returns its value.

    :param cnf: config dictionary
    :type cnf: {str, [dict]}
    :returns: list of cnf var dicts
    :rtype: [{str, int or str or None}]
    :raises: :py:class:`InvalidJsonCnf` if there is no `cnf` field found
    """
    cnf_vars = cnf.get("cnf")
    if cnf_vars is None:
        raise InvalidJsonCnf("toplevel \"cnf\" field required")
    return cnf_vars


def gen_tmpname():
    """
    Get a (quite) safe temporary file name for config file.

    :returns: temporary file name
    :rtype: str
    """
    now = time.time()
    file_handle, file_name = tempfile.mkstemp(prefix="simple_%d_" % int(now),
                                              suffix=".cnf")
    os.close(file_handle)
    os.unlink(file_name)
    return file_name


def set_values(cnf_vars, replacements):
    """
    Recursively replace values in configuration

    Works in-place, meaning that no new configuration is created and returned
    but instead argument `cnf_vars` is modified (and nothing returned).

    :param cnf_vars: config where replacements are to be made
    :type cnf_vars: [{str, int or str or None}] or {str, [dict]}
    :param replacements: what to replace and what to replace it with
    :type replacements: {str, str} or [(str, str)]
    :raises: :py:class:`InvalidJsonCnf` if cnf_vars is neither dict or list
    """
    # determine set replace_me of keys to replace and function get that returns
    # value for key or empty string if key not in replacements
    replace_me = None
    get = None
    if isinstance(replacements, dict):
        replace_me = set(k.lower() for k in replacements.keys())
        get = lambda var: str(replacements.get(var, ""))  # pylint: disable=function-redefined
    elif isinstance(replacements, list):
        replace_me = set(r[0].lower() for r in replacements)

        def get(var):                      # pylint: disable=function-redefined
            """Get replacement value for given variable name."""
            try:
                return str(next(r[1] for r in replacements if r[0] == var))
            except StopIteration:
                return ""
    else:
        raise TypeError("replacements must be dictionary or key-value list")

    # check type of arg "cnf_vars"
    if isinstance(cnf_vars, dict):
        cnf_vars = cnf_vars["cnf"]  # operate on the var list
    if not isinstance(cnf_vars, list):
        raise InvalidJsonCnf("ill-formed CNF_VAR: expected list, got %s (%s)"
                             % (type(cnf_vars), cnf_vars))

    def aux(varlist):
        """Internal recursive function to replace values."""
        for var in varlist:
            varname = var["varname"].lower()
            if varname in replace_me:
                var["data"] = str(get(varname))
            children = var.get("children", None)
            if children is not None:
                aux(children)

    # apply function on complete cnf_vars
    aux(cnf_vars)


def lookup_cnf_file(fname):
    """
    Searches for config file with given name in default locations.

    :param str fname: file name of config file (without path)
    :returns: first existing config file found in default locations
    :rtype: str
    :raises: :py:class:`IOError` if no such config file was found
    """
    locations = [arnied_wrapper.SRC_CONFIG_DIR, ADD_CNFFILE_PATH]
    for path in locations:
        fullpath = os.path.join(path, fname)
        if os.path.isfile(fullpath):
            return fullpath
    raise IOError("config file %s does not exist in any of the readable "
                  "locations %s" % (fname, locations))


###############################################################################
#                                primary class
###############################################################################


class SimpleCnf(object):
    """
    Representation of hierarchical configuration of variables.

    Based on C++ `cnf_vars` as visualized by *get_cnf*.

    Internal data representation: see module doc
    """

    def __init__(self, cnf=None):
        """
        Creates a simple configuration.

        :param cnf: initial set of conf var data (default: None = empty conf)
        :type cnf: list or anything that :py:func:`get_cnf` can read
        """
        if cnf is None:
            self.__cnfvars = []
        elif isinstance(cnf, list):
            self.__cnfvars = cnf
        elif isinstance(cnf, dict):
            self.__cnfvars = get_cnf(cnf)
        else:
            raise InvalidCnf ("cannot handle %s type inputs" % type (cnf))

    def _find_new_number(self, cnf_vars):
        """Recursive helper function to find new unique (line) number."""
        if not cnf_vars:
            return 1
        new_numbers = [1, ]   # in case cnf_vars is empty
        for cnf_var in cnf_vars:
            new_numbers.append(cnf_var['number'] + 1)
            try:
                new_numbers.append(self._find_new_number(cnf_var['children']))
            except KeyError:
                pass
        return max(new_numbers)          # this is max(all numbers) + 1

    def _find_new_instance(self, varname):
        """
        Find an instance number for variable with non-unique varname.

        Will only check on top level, is not recursive.

        :param str varname: name of conf var; will be converted to lower-case
        :returns: instance number for which there is no other conf var of same
                  name (0 if there is not other conf var with that name)
        :rtype: int
        """
        result = 0
        varname = varname.lower()
        for entry in self.__cnfvars:
            if entry['varname'] == varname:
                result = max(result, entry['number']+1)
        return result

    def add(self, varname, data='', number=None, instance=None, children=None):
        """
        Add a cnf var to config on top level.

        :param str varname: name of conf var; only required arg; case ignored
        :param str data: conf var's value
        :param int number: line number of that conf var; if given as None
                           (default) the function looks through config to find
                           a new number that is not taken; must be positive!
                           Value will be ignored if children are given.
        :param int instance: Instance of the new conf var or None (default).
                             If None, then function looks through config to
                             find a new unique instance number
        :param children: child confs for given conf var. Children's parent
                         and line attributes will be set in this function
        :type children: :py:class:`SimpleCnf`
        """
        if instance is None:
            instance = self._find_new_instance(varname)
        if children:
            number = self._find_new_number(self.__cnfvars)  # need top number
            new_children = []
            for child in children:
                new_dict = child.get_single_dict()
                new_dict['parent'] = number
                new_children.append(new_dict)
            cnfvar.renumber_vars({'cnf':new_children}, number)
            children = new_children
        elif number is None:
            number = self._find_new_number(self.__cnfvars)

        new_var = dict(varname=varname.lower(), data=data,
                       number=number, comment=None, instance=instance)
        if children:
            new_var['children'] = children   # only add if non-empty
        self.__cnfvars.append(new_var)


    def add_single(self, varname, data=u'', number=None):
        """
        Add a single cnf var to config on top level.

        Compatibility API.
        """
        return self.add (varname, data=data, number=number)


    def append_file_generic(self, reader, cnf, replacements=None):
        """
        Append conf var data from file.

        If `replacements` are given, calls :py:meth:`set_values` with these
        before adding values to config.

        :param cnf: file name or dictionary of conf vars
        :type cnf: str or {str, int or str or None}
        :param replacements: see help in :py:meth:`set_values`
        """
        log.info("append CNF_VARs from file")
        new_vars = None
        if callable(reader) is False:
            raise TypeError("append_file_generic: reader must be callable, "
                            "not %s" % type(reader))
        if isinstance(cnf, dict):
            new_vars = get_cnf(cnf)
        elif isinstance(cnf, str):
            fullpath = lookup_cnf_file(cnf)
            with open(fullpath, "rb") as chan:
                cnfdata = chan.read()
                tmp = reader(cnfdata)
                new_vars = get_cnf(tmp)
        if new_vars is None:
            raise InvalidCnf("Cannot append object \"%s\" of type \"%s\"."
                             % (cnf, type(cnf)))

        if replacements is not None:
            set_values(new_vars, replacements)

        self.__cnfvars.extend(new_vars)

    def append_file(self, cnf, replacements=None):
        """Append conf var data from file."""
        return self.append_file_generic(cnfvar.read_cnf, cnf,
                                        replacements=replacements)

    def append_file_json(self, cnf, replacements=None):
        """Append conf var data from json file."""
        return self.append_file_generic(cnfvar.read_cnf_json, cnf,
                                        replacements=replacements)

    def append_guest_vars(self, vm=None, varname=None, replacements=None):
        """
        Append content from machine's "real" config to this object.

        Runs `get_cnf -j [varname]` on local host or VM (depending on arg
        `vm`), converts output and appends it to this objects' conf var set.
        If replacements are given, runs :py:meth:`set_values`, first.

        :param vm: a guest vm or None to run on local host
        :type vm: VM object or None
        :param str varname: optional root of conf vars to append. If given as
                            None (default), append complete conf
        :param replacements: see help in :py:meth:`set_values`
        """
        cnf = arnied_wrapper.get_cnfvar(varname=varname, vm=vm)
        new_vars = get_cnf(cnf)

        log.info("apply substitutions to extracted CNF_VARs")
        if replacements is not None:
            set_values(new_vars, replacements)

        current = self.__cnfvars
        current.extend(new_vars)

    def save(self, filename=None):
        """
        Saves this object's configuration data to a file.

        The output file's content can be interpreted by `set_cnf -j`.

        :param str filename: name of file to write config to; if None (default)
                             the config will be written to a temporary file
        :returns: filename that was written to
        :rtype: str
        """
        log.info("save configuration")
        current = self.__cnfvars
        if not current:
            raise InvalidCnf("No variables to write.")

        if filename is None:
            # create temporary filename
            filename = arnied_wrapper.generate_config_path(dumped=True)

        with open(filename, 'w') as out:
            cnfvar.output_json({"cnf": current}, out, renumber=True)

        return filename

    def apply(self, vm=None, renumber=True):
        """
        Apply object's config on VM or local host.

        Runs a `set_cnf` with complete internal config data, possibly waits for
        generate to finish afterwards.

        :param vm: a guest vm or None to apply on local host
        :type vm: VM object or None
        :param bool renumber: re-number conf vars before application
        """
        current = self.__cnfvars
        if renumber:
            log.info("enforce consistent CNF_LINE numbering")
            cnfvar.renumber_vars(current)
        log.info("inject configuration %s" % "into guest" if vm else "in place")
        arnied_wrapper.set_cnf_dynamic({"cnf": current},
                                       config_file=gen_tmpname(), vm=vm)

    def __str__(self):
        """
        Get a config in json format, ready for `set_cnf -j`.

        :returns: config in json format
        :rtype: str
        """
        return cnfvar.dump_json_string({"cnf": self.__cnfvars}, renumber=True)

    def pretty_print(self, print_func=None):
        """
        Get a string representation of this simple_cnf that is human-readable

        Result is valid json with nice line breaks and indentation but not
        renumbered (so may not be fit for parsing)
        """
        for line in json.dumps({"cnf": self.__cnfvars}, check_circular=False,
                               indent=4, sort_keys=True).splitlines():
            if print_func is None:
                print(line)
            else:
                print_func(line)

    def __iter__(self):
        """
        Return an iterator over the contents of this simple cnf.

        The iteration might not be ordered by line number nor entry nor
        anything else. No guarantees made!

        The entries returned by the iterator are :py:class:`SimpleCnf`.

        Example::

            for cnf_list in iter(my_cnf['PROXY_ACCESSLIST']):
                print('checking proxy list {0} with {1} children'
                      .format(cnf_list.get_value(), len(cnf_list)))
        """
        # self.__cnfvars is a list of dicts, each with the same fields
        for dict_entry in self.__cnfvars:
            yield SimpleCnf([dict_entry, ])

    def __getitem__(self, item):
        """
        Called by `cnf['key']` or `cnf[line_number]`; returns subset of cnf.

        Processing time is O(n) where n is the number of top-level entries in
        simple cnf.

        Examples (on VM)::

            all = SimpleCnf()
            all.append_guest_vars()
            len(all)           # --> probably huge
            len(all['user'])   # should give the number of users

            # should result in the same as all['user']:
            users = SimpleCnf()
            users.append_guest_vars(varname='user')

        :param item: line number or value to specify a cnf subset;
                     if string value, will be converted to lower case
        :type item: int or str
        :returns: another simple cnf that contains a subset of this simple cnf
        :rtype: :py:class:`SimpleCnf`

        .. seealso:: method :py:func:`get` (more general than this)
        """
        # determine whether arg 'item' is a key name or a line number
        if isinstance(item, int):  # is line number
            dict_key = 'number'
        else:                      # assume key name
            dict_key = 'varname'
            item = item.lower()

        # search all entries for matches
        results = [dict_entry for dict_entry in self.__cnfvars
                   if dict_entry[dict_key] == item]

        # convert result to a simple cnf
        return SimpleCnf(results)

    def __len__(self):
        """
        Get the number of top-level entries in cnf.

        :returns: number of top-level entries in cnf
        :rtype: int
        """
        return len(self.__cnfvars)

    def get(self, name=None, value=None, instance=None, line=None):
        """
        Get a subset of this config that matches ALL of given criteria.

        For example, if :py:func:`get_cnf` contains the line
        '1121 USER,1: "admin"', all of these examples will result in the same
        simple cnf::

            cnf.get(name='user', value='admin')
            cnf.get(name='user', instance=1)
            cnf.get(name='user').get(value='admin')
            cnf.get(line=1121)

        :param str name: conf var name (key) or None to not use this criterion;
                         will be converted to lower case
        :param str value: value of conf var or None to not use this criterion
        :param int instance: instance number of value in a list (e.g. USERS)
                             or None to not use this criterion
        :param int line: line number of None to not use this criterion
        :returns: a simple cnf that contains only entries that match ALL of the
                  given criteria. If nothing matches the given criteria, an
                  empty simple cnf will be returned
        :rtype: :py:class:`SimpleCnf`

        .. seealso:: method :py:func:`__getitem__` (less general than this)
        """
        if name is None:
            name_test = lambda test_val: True
        else:
            name = name.lower()
            name_test = lambda test_val: name == test_val['varname']

        if value is None:
            value_test = lambda test_val: True
        else:
            value = str(value)
            value_test = lambda test_val: test_val['data'] == value

        if instance is None:
            instance_test = lambda test_val: True
        elif not isinstance(instance, int):
            raise ValueError('expect int value for instance!')
        else:
            instance_test = lambda test_val: instance == test_val['instance']

        if line is None:
            line_test = lambda test_val: True
        elif not isinstance(line, int):
            raise ValueError('expect int value for line number!')
        else:
            line_test = lambda test_val: test_val['number'] == line

        return SimpleCnf(list(entry for entry in self.__cnfvars
                               if name_test(entry) and value_test(entry)
                               and instance_test(entry) and line_test(entry)))

    def get_children(self):
        """
        Get children of simple cnf of just 1 entry.

        :returns: simple cnf children or an empty simple cnf if entry has
                  no children
        :rtype: :py:class:`SimpleCnf`
        :raises: :py:class:`ValueError` if this simple cnf has more
                 than 1 entry
        """
        if len(self) != 1:
            raise ValueError('get_children only possible if len == 1 (is {0})!'
                             .format(len(self)))
        try:
            result = self.__cnfvars[0]['children']
        except KeyError:
            return SimpleCnf()

        for entry in result:
            try:
                del entry['parent']
            except KeyError:
                pass
        return SimpleCnf(result)

    def get_value(self):
        """
        Get a value of a simple cnf of just 1 entry.

        :returns: str cnf value/data
        :rtype: str
        :raises: :py:class:`ValueError` if this simple cnf has more
                 than 1 entry
        """
        if len(self) != 1:
            raise ValueError('get_value only possible if len == 1 (is {0})!'
                             .format(len(self)))
        return self.__cnfvars[0]['data']

    def get_single_dict(self):
        """
        Get a dictionary of a simple cnf of just 1 entry.

        :returns: dictionary of a simple cnf
        :rtype: {str, int or str or None}
        """
        if len(self) != 1:
            raise ValueError('get_single_dict only possible if len == 1 (is {0})!'
                             .format(len(self)))
        return self.__cnfvars[0]

    def __eq__(self, other_cnf):
        """
        Determine wether `self` == `other_cnf`.

        :param other_cnf: cnf to compare with
        :type other_cnf: :py:class:`SimpleCnf`
        :returns: whether all cnf var entries are equal
        :rtype: bool
        """
        key_func = lambda cnf_var_entry: cnf_var_entry['number']

        if isinstance (other_cnf, SimpleCnf) is False:
            return False

        return sorted(self.__cnfvars, key=key_func) \
            == sorted(other_cnf.__cnfvars, key=key_func)   # pylint: disable=protected-access