Add a new function to wait for arnied to be ready

[pyi2ncommon] / src / arnied_wrapper.py

# The software in this package is distributed under the GNU General
# Public License version 2 (with a special exception described below).
#
# A copy of GNU General Public License (GPL) is included in this distribution,
# in the file COPYING.GPL.
#
# As a special exception, if other files instantiate templates or use macros
# or inline functions from this file, or you compile this file and link it
# with other works to produce a work based on this file, this file
# does not by itself cause the resulting work to be covered
# by the GNU General Public License.
#
# However the source code for this file must still be made available
# in accordance with section (3) of the GNU General Public License.
#
# This exception does not invalidate any other reasons why a work based
# on this file might be covered by the GNU General Public License.
#
# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>

"""

SUMMARY
------------------------------------------------------
Guest utility to wrap arnied related functionality through python calls.

Copyright: Intra2net AG


There are three types of setting some cnfvar configuration:

1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
   config file without modifying its content in any way
2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
   static file but rather as a template, replacing regex-matched values to
   adapt it to different configurations
3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
   and custom cnfvar classes and writing them into config files of a desired
   format (json, cnf, or raw)


INTERFACE
------------------------------------------------------

"""

import os
import sys
import time
import re
import subprocess
import shutil
import tempfile
import logging
log = logging.getLogger('pyi2ncommon.arnied_wrapper')

from .cnfline import build_cnfvar
from . import cnfvar
from . import sysmisc



#: default set_cnf binary
BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
#: default arnied_helper binary
BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
#: default location for template configuration files
SRC_CONFIG_DIR = "."
#: default location for dumped configuration files
DUMP_CONFIG_DIR = "."


class ConfigError(Exception):
    pass


def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
    """
    Universal command run wrapper.

    :param str cmd: command to run
    :param bool ignore_errors: whether not to raise error on command failure
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :param int timeout: amount of seconds to wait for the program to run
    :returns: command result output
    :rtype: str
    :raises: :py:class:`OSError` if command failed and cannot be ignored
    """
    if vm is not None:
        status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
        stdout = stdout.encode()
        stderr = b""
        if status != 0:
            stderr = stdout
            stdout = b""
            if not ignore_errors:
                raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
        return subprocess.CompletedProcess(cmd, status, stdout=stdout, stderr=stderr)
    else:
        return subprocess.run(cmd, check=not ignore_errors, shell=True, capture_output=True)


def verify_running(process='arnied', timeout=60, vm=None):
    """
    Verify if a given process is running via 'pgrep'.

    :param str process: process to verify if running
    :param int timeout: run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :raises: :py:class:`RuntimeError` if process is not running
    """
    platform_str = ""
    if vm is not None:
        vm.verify_alive()
        platform_str = " on %s" % vm.name
    for i in range(timeout):
        log.info("Checking whether %s is running%s (%i\%i)",
                 process, platform_str, i, timeout)
        result = run_cmd(cmd="pgrep -l -x %s" % process,
                         ignore_errors=True, vm=vm)
        if result.returncode == 0:
            log.debug(result)
            return
        time.sleep(1)
    raise RuntimeError("Process %s does not seem to be running" % process)


# Basic functionality


def accept_licence(vm=None):
    """
    Accept the Intra2net license.

    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None

    This is mostly useful for simplified webpage access.
    """
    cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
    result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
    log.debug(result)
    cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end GENERATE"
    run_cmd(cmd=cmd, vm=vm)


def go_online(provider_id, wait_online=True, timeout=60, vm=None):
    """
    Go online with the given provider id.

    :param provider_id: provider to go online with
    :type provider_id: int
    :param wait_online: whether to wait until online
    :type wait_online: bool
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None

    .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
    """
    log.info("Switching to online mode with provider %d", provider_id)

    get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
    if b'PROVIDER,' not in get_cnf_res.stdout:
        log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
                    provider_id)
        return

    cmd = 'tell-connd --online P%i' % provider_id
    result = run_cmd(cmd=cmd, vm=vm)
    log.debug(result)

    if wait_online:
        wait_for_online(provider_id, timeout=timeout, vm=vm)


def go_offline(wait_offline=True, vm=None):
    """
    Go offline.

    :param wait_offline: whether to wait until offline
    :type wait_offline: bool
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None

    .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
    """
    cmd = 'tell-connd --offline'
    result = run_cmd(cmd=cmd, vm=vm)
    log.debug(result)

    if wait_offline:
        if wait_offline is True:
            wait_for_offline(vm=vm)
        else:
            wait_for_offline(wait_offline, vm=vm)


def wait_for_offline(timeout=60, vm=None):
    """
    Wait for arnied to signal we are offline.

    :param int timeout: maximum timeout for waiting
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    _wait_for_online_status('offline', None, timeout, vm)


def wait_for_online(provider_id, timeout=60, vm=None):
    """
    Wait for arnied to signal we are online.

    :param provider_id: provider to go online with
    :type provider_id: int
    :param int timeout: maximum timeout for waiting
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    _wait_for_online_status('online', provider_id, timeout, vm)


def _wait_for_online_status(status, provider_id, timeout, vm):
    # Don't use tell-connd --status here since the actual
    # ONLINE signal to arnied is transmitted
    # asynchronously via arnieclient_muxer.

    if status == 'online':
        expected_output = 'DEFAULT: 2'
        set_status_func = lambda: go_online(provider_id, False, vm)
    elif status == 'offline':
        expected_output = 'DEFAULT: 0'
        set_status_func = lambda: go_offline(False, vm)
    else:
        raise ValueError('expect status "online" or "offline", not "{0}"!'
                         .format(status))

    log.info("Waiting for arnied to be {0} within {1} seconds"
             .format(status, timeout))

    for i in range(timeout):
        # arnied might invalidate the connd "connection barrier"
        # after generate was running and switch to OFFLINE (race condition).
        # -> tell arnied every ten seconds to go online again
        if i % 10 == 0 and i != 0:
            set_status_func()

        cmd = '/usr/intranator/bin/get_var ONLINE'
        result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
        log.debug(result)

        if expected_output in result.stdout.decode():
            log.info("arnied is {0}. Continuing.".format(status))
            return

        time.sleep(1)

    raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
                       .format(status, timeout))


def disable_virscan(vm=None):
    """
    Disable virscan that could block GENERATE and thus all configurations.

    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    log.info("Disabling virus database update")
    unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)

    cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
    result = run_cmd(cmd=cmd, vm=vm)
    log.debug(result)

    # TODO: this intervention should be solved in later arnied_helper tool
    cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
    result = run_cmd(cmd=cmd, vm=vm)
    log.debug(result)
    log.info("Virus database update disabled")


def email_transfer(vm=None):
    """
    Transfer all the emails using the guest tool arnied_helper.

    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
    result = run_cmd(cmd=cmd, vm=vm)
    log.debug(result)


def wait_for_email_transfer(timeout=300, vm=None):
    """
    Wait until the mail queue is empty and all emails are sent.

    :param int timeout: email transfer timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    for i in range(timeout):
        if i % 10 == 0:
            # Retrigger mail queue in case something is deferred
            # by an amavisd-new reconfiguration
            run_cmd(cmd='postqueue -f', vm=vm)
            log.info('Waiting for SMTP queue to get empty (%i/%i s)',
                     i, timeout)
        if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
            log.debug('SMTP queue is empty')
            return
        time.sleep(1)
    log.warning('Timeout reached but SMTP queue still not empty after {} s'
                .format(timeout))


def schedule(program, exec_time=0, optional_args="", vm=None):
    """
    Schedule a program to be executed at a given unix time stamp.

    :param str program: program whose execution is scheduled
    :param int exec_time: scheduled time of program's execution
    :param str optional_args: optional command line arguments
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    log.info("Scheduling %s to be executed at %i", program, exec_time)
    schedule_dir = "/var/intranator/schedule"
    # clean previous schedules of the same program
    files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
    for file_name in files:
        if file_name.startswith(program.upper()):
            log.debug("Removing previous scheduled %s", file_name)
            if vm:
                vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
            else:
                os.unlink(os.path.join(schedule_dir, file_name))

    contents = "%i\n%s\n" % (exec_time, optional_args)

    tmp_file = tempfile.NamedTemporaryFile(mode="w+",
                                        prefix=program.upper() + "_",
                                        dir=DUMP_CONFIG_DIR,
                                        delete=False)
    log.debug("Created temporary file %s", tmp_file.name)
    tmp_file.write(contents)
    tmp_file.close()
    moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))

    if vm:
        vm.copy_files_to(tmp_file.name, moved_tmp_file)
        os.remove(tmp_file.name)
    else:
        shutil.move(tmp_file.name, moved_tmp_file)

    log.debug("Moved temporary file to %s", moved_tmp_file)


def wait_for_run(program, timeout=300, retries=10, vm=None):
    """
    Wait for a program using the guest arnied_helper tool.

    :param str program: scheduled or running program to wait for
    :param int timeout: program run timeout
    :param int retries: number of tries to verify that the program is scheduled or running
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    log.info("Waiting for program %s to finish with timeout %i",
             program, timeout)
    for i in range(retries):
        cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
            + program.upper()
        check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
        if check_scheduled.returncode == 0:
            break
        time.sleep(1)
    else:
        log.warning("The program %s was not scheduled and is not running", program)
        return
    cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
          f"{program.upper()} --wait-for-program-timeout {timeout}"
    # add one second to make sure arnied_helper is finished when we expire
    result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
    log.debug(result.stdout)


def wait_for_arnied(timeout=60, vm=None):
    """
    Wait for arnied socket to be ready.

    :param int timeout: maximum number of seconds to wait
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
          f"--wait-for-arnied-socket-timeout {timeout}"
    # add one second to make sure arnied_helper is finished when we expire
    result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
    log.debug(result.stdout)


# Configuration functionality

def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
    """
    Query arnied for a `cnf_key` and extract some information via regex.

    :param str cnf_key: queried cnf key
    :param int cnf_index: index of the cnf key
    :param str regex: regex to apply on the queried cnf key data
    :param bool compact: whether to retrieve compact version of the matched cnf keys
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :returns: extracted information via the regex
    :rtype: Match object

    If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
    """
    wait_for_arnied(timeout=timeout, vm=vm)
    platform_str = ""
    if vm is not None:
        platform_str = " from %s" % vm.name
    log.info("Extracting arnied value %s for %s%s using pattern %s",
             cnf_index, cnf_key, platform_str, regex)
    cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
                              " %s" % cnf_index if cnf_index != -1 else "")
    output = run_cmd(cmd=cmd, vm=vm).stdout.decode()
    return re.search(regex, output, flags=re.DOTALL)


def get_cnf_id(cnf_key, value, timeout=30, vm=None):
    """
    Get the id of a configuration of type `cnf_key` and name `value`.

    :param str cnf_key: queried cnf key
    :param str value: cnf value of the cnf key
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :returns: the cnf id or -1 if no such cnf variable
    :rtype: int
    """
    wait_for_arnied(timeout=timeout, vm=vm)
    regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
    cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
    if cnf_id is None:
        cnf_id = -1
    else:
        cnf_id = int(cnf_id.group(1))
    log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
    return cnf_id


def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
    """
    Invoke get_cnf and return a nested CNF structure.

    :param str varname: "varname" field of the CNF_VAR to look up
    :param instance: "instance" of that variable to return
    :type instance: int
    :param str data: "data" field by which the resulting CNF_VAR list should be filtered
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
    :rtype: cnfvar option
    """
    wait_for_arnied(timeout=timeout, vm=vm)
    # firstly, build argv for get_cnf
    cmd = ["get_cnf", "-j"]
    if varname is not None:
        cmd.append("%s" % varname)
        if instance:
            cmd.append("%d" % instance)
    cmd_line = " ".join(cmd)

    # now invoke get_cnf
    result = run_cmd(cmd=cmd_line, vm=vm)
    (status, raw) = result.returncode, result.stdout
    if status != 0:
        log.info("error %d executing \"%s\"", status, cmd_line)
        log.debug(raw)
        return None

    # reading was successful, attempt to parse what we got
    try:
        # The output from "get_cnf -j" is already utf-8. This contrast with
        # the output of "get_cnf" (no json) which is latin1.
        if isinstance(raw, bytes):
            raw = raw.decode("utf-8")
        cnf = cnfvar.read_cnf_json(raw)
    except TypeError as exn:
        log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
        return None
    except cnfvar.InvalidCNF as exn:
        log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
        return None

    if data is not None:
        return cnfvar.get_vars(cnf, data=data)

    return cnf


def get_cnfvar_id(varname, data, timeout=30, vm=None):
    """
    Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.

    :param str varname: "varname" field of the CNF_VAR to look up
    :param str data: "data" field by which the resulting CNF_VAR list should be filtered
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :returns: the cnf id or -1 if no such cnf variable
    :rtype: int
    """
    wait_for_arnied(timeout=timeout, vm=vm)
    log.info("Extracting from arnied CNF_VAR %s with data %s",
             varname, data)
    cnf = get_cnfvar(varname=varname, data=data, vm=vm)
    variables = cnf["cnf"]
    if len(variables) == 0:
        log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
        # preserve behavior
        return -1
    first_instance = int(variables[0]["instance"])
    log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
             len(variables), first_instance)
    return first_instance


def wait_for_generate(timeout=300, vm=None):
    """
    Wait for the 'generate' program to complete.

    Arguments are similar to the ones from :py:method:`wait_for_run`.
    """
    wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
    wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)


def unset_cnf(varname="", instance="", timeout=30, vm=None):
    """
    Remove configuration from arnied.

    :param str varname: "varname" field of the CNF_VAR to unset
    :param int instance: "instance" of that variable to unset
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    """
    wait_for_arnied(timeout=timeout, vm=vm)

    cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
    run_cmd(cmd=cmd, vm=vm)

    wait_for_generate(vm=vm)


def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
    """
    Perform static arnied configuration through a set of config files.

    :param config_files: config files to use for the configuration
    :type config_files: [str]
    :param str kind: "json" or "cnf"
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :raises: :py:class:`ConfigError` if cannot apply file

    The config files must be provided and are always expected to be found on
    the host. If these are absolute paths, they will be kept as is or
    otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
    the config files will be copied there as temporary files before applying.
    """
    log.info("Setting arnied configuration")
    wait_for_arnied(timeout=timeout, vm=vm)

    config_paths = prep_config_paths(config_files)
    for config_path in config_paths:
        with open(config_path, "rt", errors='replace') as config:
            log.debug("Contents of applied %s:\n%s", config_path, config.read())
        if vm is not None:
            new_config_path = generate_config_path()
            vm.copy_files_to(config_path, new_config_path)
            config_path = new_config_path
        argv = ["set_cnf", kind == "json" and "-j" or "", config_path]

        result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
        logging.debug(result)
        if result.returncode != 0:
            raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
                              % (config_path,
                                 " on %s" % vm.name if vm is not None else "",
                                 result.returncode))

    try:
        wait_for_generate(vm=vm)
    except Exception as ex:
        # handle cases of remote configuration that leads to connection meltdown
        if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
            log.info("Resetting connection to %s", vm.name)
            vm.session = vm.wait_for_login(timeout=10)
            log.debug("Connection reset via remote error: %s", ex)
        else:
            raise ex


def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
                        kind="cnf", timeout=30, vm=None):
    """
    Perform semi-dynamic arnied configuration from an updated version of the
    config files.

    :param config_files: config files to use for the configuration
    :type config_files: [str]
    :param params_dict: parameters to override the defaults in the config files
    :type params_dict: {str, str}
    :param regex_dict: regular expressions to use for matching the overriden parameters
    :type regex_dict: {str, str} or None
    :param str kind: "json" or "cnf"
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None

    The config files must be provided and are always expected to be found on
    the host. If these are absolute paths, they will be kept as is or
    otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
    the config files will be copied there as temporary files before applying.
    """
    log.info("Performing semi-dynamic arnied configuration")

    config_paths = prep_cnf(config_files, params_dict, regex_dict)
    set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)

    log.info("Semi-dynamic arnied configuration successful!")


def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
    """
    Perform dynamic arnied configuration from fully generated config files.

    :param cnf: one key with the same value as *kind* and a list of cnfvars as value
    :type cnf: {str, str}
    :param config_file: optional user supplied filename
    :type config_file: str or None
    :param str kind: "json", "cnf", or "raw"
    :param int timeout: arnied run verification timeout
    :param vm: vm to run on if running on a guest instead of the host
    :type vm: VM object or None
    :raises: :py:class:`ValueError` if `kind` is not an acceptable value
    :raises: :py:class:`ConfigError` if cannot apply file

    The config file might not be provided in which case a temporary file will
    be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
    an absolute path. If a vm is provided, the config file will be copied there
    as a temporary file before applying.
    """
    if config_file is None:
        config_path = generate_config_path(dumped=True)
    elif os.path.isabs(config_file):
        config_path = config_file
    else:
        config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
    generated = config_file is None
    config_file = os.path.basename(config_path)
    log.info("Using %s cnf file %s%s",
             "generated" if generated else "user-supplied",
             config_file, " on %s" % vm.name if vm is not None else "")

    # Important to write bytes here to ensure text is encoded with latin-1
    fd = open(config_path, "wb")
    try:
        SET_CNF_METHODS = {
            "raw": cnfvar.write_cnf_raw,
            "json": cnfvar.write_cnf_json,
            "cnf": cnfvar.write_cnf
        }
        SET_CNF_METHODS[kind](cnf, out=fd)
    except KeyError:
        raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
                         % kind)
    finally:
        fd.close()
    log.info("Generated config file %s", config_path)

    kind = "cnf" if kind != "json" else kind
    set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)


def set_cnf_pipe(cnf, timeout=30, block=False):
    """
    Set local configuration by talking to arnied via ``set_cnf``.

    :param cnf: one key with the same value as *kind* and a list of cnfvars as value
    :type cnf: {str, str}
    :param int timeout: arnied run verification timeout
    :param bool block: whether to wait for generate to complete the
                       configuration change
    :returns: whether ``set_cnf`` succeeded or not
    :rtype: bool

    This is obviously not generic but supposed to be run on the guest.
    """
    log.info("Setting arnied configuration through local pipe")
    wait_for_arnied(timeout=timeout)

    st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))

    if st is False:
        log.error("Error applying configuration; status=%r" % exit)
        log.error("and stderr:\n%s" % out)
        return False
    log.debug("Configuration successfully passed to set_cnf, "
              "read %d B from pipe" % len(out))

    if block is True:
        log.debug("Waiting for config job to complete")
        wait_for_generate()

    log.debug("Exiting sucessfully")
    return True


def prep_config_paths(config_files, config_dir=None):
    """
    Prepare absolute paths for all configs at an expected location.

    :param config_files: config files to use for the configuration
    :type config_files: [str]
    :param config_dir: config directory to prepend to the filepaths
    :type config_dir: str or None
    :returns: list of the full config paths
    :rtype: [str]
    """
    if config_dir is None:
        config_dir = SRC_CONFIG_DIR
    config_paths = []
    for config_file in config_files:
        if os.path.isabs(config_file):
            # Absolute path: The user requested a specific file
            # f.e. needed for dynamic arnied config update
            config_path = config_file
        else:
            config_path = os.path.join(os.path.abspath(config_dir),
                                       config_file)
        logging.debug("Using %s for original path %s", config_path, config_file)
        config_paths.append(config_path)
    return config_paths


def prep_cnf_value(config_file, value,
                   regex=None, template_key=None, ignore_fail=False):
    """
    Replace value in a provided arnied config file.

    :param str config_file: file to use for the replacement
    :param str value: value to replace the first matched group with
    :param regex: regular expression to use when replacing a cnf value
    :type regex: str or None
    :param template_key: key of a quick template to use for the regex
    :type template_key: str or None
    :param bool ignore_fail: whether to ignore regex mismatching
    :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match

    In order to ensure better matching capabilities you are supposed to
    provide a regex pattern with at least one subgroup to match your value.
    What this means is that the value you like to replace is not directly
    searched into the config text but matched within a larger regex in
    in order to avoid any mismatch.

    Example:
    provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
    """
    if template_key is None:
        pattern = regex.encode()
    else:
        samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
                   "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
        pattern = samples[template_key].encode()

    with open(config_file, "rb") as file_handle:
        text = file_handle.read()
    match_line = re.search(pattern, text)

    if match_line is None and not ignore_fail:
        raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
    elif match_line is not None:
        old_line = match_line.group(0)
        text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
        line = re.search(pattern, text).group(0)
        log.debug("Updating %s to %s in %s", old_line, line, config_file)
        with open(config_file, "wb") as file_handle:
            file_handle.write(text)


def prep_cnf(config_files, params_dict, regex_dict=None):
    """
    Update all config files with the default overriding parameters,
    i.e. override the values hard-coded in those config files.

    :param config_files: config files to use for the configuration
    :type config_files: [str]
    :param params_dict: parameters to override the defaults in the config files
    :type params_dict: {str, str}
    :param regex_dict: regular expressions to use for matching the overriden parameters
    :type regex_dict: {str, str} or None
    :returns: list of prepared (modified) config paths
    :rtype: [str]
    """
    log.info("Preparing %s template config files", len(config_files))

    src_config_paths = prep_config_paths(config_files)
    new_config_paths = []
    for config_path in src_config_paths:
        new_config_path = generate_config_path(dumped=True)
        shutil.copy(config_path, new_config_path)
        new_config_paths.append(new_config_path)

    for config_path in new_config_paths:
        for param_key in params_dict.keys():
            if regex_dict is None:
                regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
            elif param_key in regex_dict.keys():
                regex_val = regex_dict[param_key] % param_key.upper()
            elif re.match("\w*_\d+$", param_key):
                final_parameter, parent_id = \
                    re.match("(\w*)_(\d+)$", param_key).group(1, 2)
                regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
                    % (parent_id, final_parameter.upper())
                log.debug("Requested regex for %s is '%s'",
                          param_key, regex_val)
            else:
                regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
            prep_cnf_value(config_path, params_dict[param_key],
                           regex=regex_val, ignore_fail=True)
        log.info("Prepared template config file %s", config_path)

    return new_config_paths


def generate_config_path(dumped=False):
    """
    Generate path for a temporary config name.

    :param bool dumped: whether the file should be in the dump
                        directory or in temporary directory
    :returns: generated config file path
    :rtype: str
    """
    dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
    fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
    os.close(fd)
    os.unlink(filename)
    return filename


# enum
Delete = 0
Update = 1
Add = 2
Child = 3


def batch_update_cnf(cnf, vars):
    """
    Perform a batch update of multiple cnf variables.

    :param cnf: CNF variable to update
    :type cnf: BuildCnfVar object
    :param vars: tuples of enumerated action and subtuple with data
    :type vars: [(int, (str, int, str))]
    :returns: updated CNF variable
    :rtype: BuildCnfVar object

    The actions are indexed in the same order: delete, update, add, child.
    """
    last = 0
    for (action, data) in vars:
        if action == Update:
            var, ref, val = data
            last = cnf.update_cnf(var, ref, val)
        elif action == Add:
            var, ref, val = data
            last = cnf.add_cnf(var, ref, val)
        elif action == Delete:
            last = cnf.del_cnf(data)
        elif action == Child:  # only one depth supported
            var, ref, val = data
            # do not update last
            cnf.add_cnf(var, ref, val, different_parent_line_no=last)
    return cnf


def build_cnf(kind, instance=0, vals=[], data="", filename=None):
    """
    Build a CNF variable and save it in a config file.

    :param str kind: name of the CNF variable
    :param int instance: instance number of the CNF variable
    :param vals: tuples of enumerated action and subtuple with data
    :type vals: [(int, (str, int, str))]
    :param str data: data for the CNF variable
    :param filename: optional custom name of the config file
    :type filename: str or None
    :returns: name of the saved config file
    :rtype: str
    """
    builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
    batch_update_cnf(builder, vals)
    filename = generate_config_path(dumped=True) if filename is None else filename
    [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
    builder.save(filename)
    return filename