SUMMARY
------------------------------------------------------
-Guest utility to wrap arnied related functionality through python calls.
+Interaction with central arnied daemon.
-.. note:: Partially DEPRECATED! Use :py:mod:`pyi2ncommon.arnied_api` or
- :py:mod:`pyi2ncommon.cnfvar` whenever possible. In particluar, do not
- use or extend functionality regarding configuration (`get_cnf`,
- `set_cnf`).
+All functions (except :py:func:`schedule` result in calling a binary
+(either :py:data:`BIN_ARNIED_HELPER` or *tell-connd*).
-Copyright: Intra2net AG
-
-There are three types of setting some cnfvar configuration:
+For changes of configuration (*set_cnf*, *get_cnf*), refer to :py:mod:`pyi2ncommon.cnfvar`.
-1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
- config file without modifying its content in any way
-2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
- static file but rather as a template, replacing regex-matched values to
- adapt it to different configurations
-3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
- and custom cnfvar classes and writing them into config files of a desired
- format (json, cnf, or raw)
+Copyright: Intra2net AG
INTERFACE
import logging
log = logging.getLogger('pyi2ncommon.arnied_wrapper')
-from .cnfline import build_cnfvar
-from . import cnfvar_old
-from . import sysmisc
-
-
-#: default set_cnf binary
-BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
#: default arnied_helper binary
BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
-#: default location for template configuration files
-SRC_CONFIG_DIR = "."
-#: default location for dumped configuration files
-DUMP_CONFIG_DIR = "."
-
-
-class ConfigError(Exception):
- pass
def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
# Basic functionality
-def accept_licence(vm=None):
- """
- Accept the Intra2net license.
-
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
-
- This is mostly useful for simplified webpage access.
- """
- cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
- result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
- log.debug(result)
- wait_for_generate(vm=vm)
-
-
def go_online(provider_id, wait_online=True, timeout=60, vm=None):
"""
Go online with the given provider id.
"""
log.info("Switching to online mode with provider %d", provider_id)
- get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
- if b'PROVIDER,' not in get_cnf_res.stdout:
- log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
- provider_id)
- return
-
cmd = 'tell-connd --online P%i' % provider_id
result = run_cmd(cmd=cmd, vm=vm)
log.debug(result)
.format(status, timeout))
-def disable_virscan(vm=None):
- """
- Disable virscan that could block GENERATE and thus all configurations.
-
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- """
- log.info("Disabling virus database update")
- unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
-
- cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
- result = run_cmd(cmd=cmd, vm=vm)
- log.debug(result)
-
- # TODO: this intervention should be solved in later arnied_helper tool
- cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
- result = run_cmd(cmd=cmd, vm=vm)
- log.debug(result)
- log.info("Virus database update disabled")
-
-
def email_transfer(vm=None):
"""
Transfer all the emails using the guest tool arnied_helper.
tmp_file = tempfile.NamedTemporaryFile(mode="w+",
prefix=program.upper() + "_",
- dir=DUMP_CONFIG_DIR,
delete=False)
log.debug("Created temporary file %s", tmp_file.name)
tmp_file.write(contents)
# Configuration functionality
-def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
- """
- Query arnied for a `cnf_key` and extract some information via regex.
-
- :param str cnf_key: queried cnf key
- :param int cnf_index: index of the cnf key
- :param str regex: regex to apply on the queried cnf key data
- :param bool compact: whether to retrieve compact version of the matched cnf keys
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :returns: extracted information via the regex
- :rtype: Match object
-
- If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
- """
- wait_for_arnied(timeout=timeout, vm=vm)
- platform_str = ""
- if vm is not None:
- platform_str = " from %s" % vm.name
- log.info("Extracting arnied value %s for %s%s using pattern %s",
- cnf_index, cnf_key, platform_str, regex)
- cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
- " %s" % cnf_index if cnf_index != -1 else "")
- # get_cnf creates latin1-encoded output, transfer from VM removes non-ascii
- output = run_cmd(cmd=cmd, vm=vm).stdout.decode('latin1')
- return re.search(regex, output, flags=re.DOTALL)
-
-
-def get_cnf_id(cnf_key, value, timeout=30, vm=None):
- """
- Get the id of a configuration of type `cnf_key` and name `value`.
-
- :param str cnf_key: queried cnf key
- :param str value: cnf value of the cnf key
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :returns: the cnf id or -1 if no such cnf variable
- :rtype: int
- """
- wait_for_arnied(timeout=timeout, vm=vm)
- regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
- cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
- if cnf_id is None:
- cnf_id = -1
- else:
- cnf_id = int(cnf_id.group(1))
- log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
- return cnf_id
-
-
-def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
- """
- Invoke get_cnf and return a nested CNF structure.
-
- :param str varname: "varname" field of the CNF_VAR to look up
- :param instance: "instance" of that variable to return
- :type instance: int
- :param str data: "data" field by which the resulting CNF_VAR list should be filtered
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
- :rtype: cnfvar option
- """
- wait_for_arnied(timeout=timeout, vm=vm)
- # firstly, build argv for get_cnf
- cmd = ["get_cnf", "-j"]
- if varname is not None:
- cmd.append("%s" % varname)
- if instance:
- cmd.append("%d" % instance)
- cmd_line = " ".join(cmd)
-
- # now invoke get_cnf
- result = run_cmd(cmd=cmd_line, vm=vm)
- (status, raw) = result.returncode, result.stdout
- if status != 0:
- log.info("error %d executing \"%s\"", status, cmd_line)
- log.debug(raw)
- return None
-
- # reading was successful, attempt to parse what we got
- try:
- # The output from "get_cnf -j" is already utf-8. This contrast with
- # the output of "get_cnf" (no json) which is latin1.
- if isinstance(raw, bytes):
- raw = raw.decode("utf-8")
- cnf = cnfvar_old.read_cnf_json(raw)
- except TypeError as exn:
- log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
- return None
- except cnfvar_old.InvalidCNF as exn:
- log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
- return None
-
- if data is not None:
- return cnfvar_old.get_vars(cnf, data=data)
-
- return cnf
-
-
-def get_cnfvar_id(varname, data, timeout=30, vm=None):
- """
- Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
-
- :param str varname: "varname" field of the CNF_VAR to look up
- :param str data: "data" field by which the resulting CNF_VAR list should be filtered
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :returns: the cnf id or -1 if no such cnf variable
- :rtype: int
- """
- wait_for_arnied(timeout=timeout, vm=vm)
- log.info("Extracting from arnied CNF_VAR %s with data %s",
- varname, data)
- cnf = get_cnfvar(varname=varname, data=data, vm=vm)
- variables = cnf["cnf"]
- if len(variables) == 0:
- log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
- # preserve behavior
- return -1
- first_instance = int(variables[0]["instance"])
- log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
- len(variables), first_instance)
- return first_instance
-
def wait_for_generate(timeout=300, vm=None):
"""
"""
wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
-
-
-def unset_cnf(varname="", instance="", timeout=30, vm=None):
- """
- Remove configuration from arnied.
-
- :param str varname: "varname" field of the CNF_VAR to unset
- :param int instance: "instance" of that variable to unset
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- """
- wait_for_arnied(timeout=timeout, vm=vm)
-
- cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
- run_cmd(cmd=cmd, vm=vm)
-
- wait_for_generate(vm=vm)
-
-
-def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
- """
- Perform static arnied configuration through a set of config files.
-
- :param config_files: config files to use for the configuration
- :type config_files: [str]
- :param str kind: "json" or "cnf"
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :raises: :py:class:`ConfigError` if cannot apply file
-
- The config files must be provided and are always expected to be found on
- the host. If these are absolute paths, they will be kept as is or
- otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
- the config files will be copied there as temporary files before applying.
- """
- log.info("Setting arnied configuration")
- wait_for_arnied(timeout=timeout, vm=vm)
-
- config_paths = prep_config_paths(config_files)
- for config_path in config_paths:
- with open(config_path, "rt", errors='replace') as config:
- log.debug("Contents of applied %s:\n%s", config_path, config.read())
- if vm is not None:
- new_config_path = generate_config_path()
- vm.copy_files_to(config_path, new_config_path)
- config_path = new_config_path
- argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
-
- result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
- logging.debug(result)
- if result.returncode != 0:
- raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
- % (config_path,
- " on %s" % vm.name if vm is not None else "",
- result.returncode))
-
- try:
- wait_for_generate(vm=vm)
- except Exception as ex:
- # handle cases of remote configuration that leads to connection meltdown
- if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
- log.info("Resetting connection to %s", vm.name)
- vm.session = vm.wait_for_login(timeout=10)
- log.debug("Connection reset via remote error: %s", ex)
- else:
- raise ex
-
-
-def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
- kind="cnf", timeout=30, vm=None):
- """
- Perform semi-dynamic arnied configuration from an updated version of the
- config files.
-
- :param config_files: config files to use for the configuration
- :type config_files: [str]
- :param params_dict: parameters to override the defaults in the config files
- :type params_dict: {str, str}
- :param regex_dict: regular expressions to use for matching the overriden parameters
- :type regex_dict: {str, str} or None
- :param str kind: "json" or "cnf"
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
-
- The config files must be provided and are always expected to be found on
- the host. If these are absolute paths, they will be kept as is or
- otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
- the config files will be copied there as temporary files before applying.
- """
- log.info("Performing semi-dynamic arnied configuration")
-
- config_paths = prep_cnf(config_files, params_dict, regex_dict)
- set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
-
- log.info("Semi-dynamic arnied configuration successful!")
-
-
-def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
- """
- Perform dynamic arnied configuration from fully generated config files.
-
- :param cnf: one key with the same value as *kind* and a list of cnfvars as value
- :type cnf: {str, str}
- :param config_file: optional user supplied filename
- :type config_file: str or None
- :param str kind: "json", "cnf", or "raw"
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :raises: :py:class:`ValueError` if `kind` is not an acceptable value
- :raises: :py:class:`ConfigError` if cannot apply file
-
- The config file might not be provided in which case a temporary file will
- be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
- an absolute path. If a vm is provided, the config file will be copied there
- as a temporary file before applying.
- """
- if config_file is None:
- config_path = generate_config_path(dumped=True)
- elif os.path.isabs(config_file):
- config_path = config_file
- else:
- config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
- generated = config_file is None
- config_file = os.path.basename(config_path)
- log.info("Using %s cnf file %s%s",
- "generated" if generated else "user-supplied",
- config_file, " on %s" % vm.name if vm is not None else "")
-
- # Important to write bytes here to ensure text is encoded with latin-1
- fd = open(config_path, "wb")
- try:
- SET_CNF_METHODS = {
- "raw": cnfvar_old.write_cnf_raw,
- "json": cnfvar_old.write_cnf_json,
- "cnf": cnfvar_old.write_cnf
- }
- SET_CNF_METHODS[kind](cnf, out=fd)
- except KeyError:
- raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
- % kind)
- finally:
- fd.close()
- log.info("Generated config file %s", config_path)
-
- kind = "cnf" if kind != "json" else kind
- set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
-
-
-def set_cnf_pipe(cnf, timeout=30, block=False):
- """
- Set local configuration by talking to arnied via ``set_cnf``.
-
- :param cnf: one key with the same value as *kind* and a list of cnfvars as value
- :type cnf: {str, str}
- :param int timeout: arnied run verification timeout
- :param bool block: whether to wait for generate to complete the
- configuration change
- :returns: whether ``set_cnf`` succeeded or not
- :rtype: bool
-
- This is obviously not generic but supposed to be run on the guest.
- """
- log.info("Setting arnied configuration through local pipe")
- wait_for_arnied(timeout=timeout)
-
- st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
-
- if st is False:
- log.error("Error applying configuration; status=%r" % exit)
- log.error("and stderr:\n%s" % out)
- return False
- log.debug("Configuration successfully passed to set_cnf, "
- "read %d B from pipe" % len(out))
-
- if block is True:
- log.debug("Waiting for config job to complete")
- wait_for_generate()
-
- log.debug("Exiting sucessfully")
- return True
-
-
-def prep_config_paths(config_files, config_dir=None):
- """
- Prepare absolute paths for all configs at an expected location.
-
- :param config_files: config files to use for the configuration
- :type config_files: [str]
- :param config_dir: config directory to prepend to the filepaths
- :type config_dir: str or None
- :returns: list of the full config paths
- :rtype: [str]
- """
- if config_dir is None:
- config_dir = SRC_CONFIG_DIR
- config_paths = []
- for config_file in config_files:
- if os.path.isabs(config_file):
- # Absolute path: The user requested a specific file
- # f.e. needed for dynamic arnied config update
- config_path = config_file
- else:
- config_path = os.path.join(os.path.abspath(config_dir),
- config_file)
- logging.debug("Using %s for original path %s", config_path, config_file)
- config_paths.append(config_path)
- return config_paths
-
-
-def prep_cnf_value(config_file, value,
- regex=None, template_key=None, ignore_fail=False):
- """
- Replace value in a provided arnied config file.
-
- :param str config_file: file to use for the replacement
- :param str value: value to replace the first matched group with
- :param regex: regular expression to use when replacing a cnf value
- :type regex: str or None
- :param template_key: key of a quick template to use for the regex
- :type template_key: str or None
- :param bool ignore_fail: whether to ignore regex mismatching
- :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
-
- In order to ensure better matching capabilities you are supposed to
- provide a regex pattern with at least one subgroup to match your value.
- What this means is that the value you like to replace is not directly
- searched into the config text but matched within a larger regex in
- in order to avoid any mismatch.
-
- Example:
- provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
- """
- if template_key is None:
- pattern = regex.encode()
- else:
- samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
- "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
- pattern = samples[template_key].encode()
-
- with open(config_file, "rb") as file_handle:
- text = file_handle.read()
- match_line = re.search(pattern, text)
-
- if match_line is None and not ignore_fail:
- raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
- elif match_line is not None:
- old_line = match_line.group(0)
- text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
- line = re.search(pattern, text).group(0)
- log.debug("Updating %s to %s in %s", old_line, line, config_file)
- with open(config_file, "wb") as file_handle:
- file_handle.write(text)
-
-
-def prep_cnf(config_files, params_dict, regex_dict=None):
- """
- Update all config files with the default overriding parameters,
- i.e. override the values hard-coded in those config files.
-
- :param config_files: config files to use for the configuration
- :type config_files: [str]
- :param params_dict: parameters to override the defaults in the config files
- :type params_dict: {str, str}
- :param regex_dict: regular expressions to use for matching the overriden parameters
- :type regex_dict: {str, str} or None
- :returns: list of prepared (modified) config paths
- :rtype: [str]
- """
- log.info("Preparing %s template config files", len(config_files))
-
- src_config_paths = prep_config_paths(config_files)
- new_config_paths = []
- for config_path in src_config_paths:
- new_config_path = generate_config_path(dumped=True)
- shutil.copy(config_path, new_config_path)
- new_config_paths.append(new_config_path)
-
- for config_path in new_config_paths:
- for param_key in params_dict.keys():
- if regex_dict is None:
- regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
- elif param_key in regex_dict.keys():
- regex_val = regex_dict[param_key] % param_key.upper()
- elif re.match("\w*_\d+$", param_key):
- final_parameter, parent_id = \
- re.match("(\w*)_(\d+)$", param_key).group(1, 2)
- regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
- % (parent_id, final_parameter.upper())
- log.debug("Requested regex for %s is '%s'",
- param_key, regex_val)
- else:
- regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
- prep_cnf_value(config_path, params_dict[param_key],
- regex=regex_val, ignore_fail=True)
- log.info("Prepared template config file %s", config_path)
-
- return new_config_paths
-
-
-def generate_config_path(dumped=False):
- """
- Generate path for a temporary config name.
-
- :param bool dumped: whether the file should be in the dump
- directory or in temporary directory
- :returns: generated config file path
- :rtype: str
- """
- dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
- fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
- os.close(fd)
- os.unlink(filename)
- return filename
-
-
-# enum
-Delete = 0
-Update = 1
-Add = 2
-Child = 3
-
-
-def batch_update_cnf(cnf, vars):
- """
- Perform a batch update of multiple cnf variables.
-
- :param cnf: CNF variable to update
- :type cnf: BuildCnfVar object
- :param vars: tuples of enumerated action and subtuple with data
- :type vars: [(int, (str, int, str))]
- :returns: updated CNF variable
- :rtype: BuildCnfVar object
-
- The actions are indexed in the same order: delete, update, add, child.
- """
- last = 0
- for (action, data) in vars:
- if action == Update:
- var, ref, val = data
- last = cnf.update_cnf(var, ref, val)
- elif action == Add:
- var, ref, val = data
- last = cnf.add_cnf(var, ref, val)
- elif action == Delete:
- last = cnf.del_cnf(data)
- elif action == Child: # only one depth supported
- var, ref, val = data
- # do not update last
- cnf.add_cnf(var, ref, val, different_parent_line_no=last)
- return cnf
-
-
-def build_cnf(kind, instance=0, vals=[], data="", filename=None):
- """
- Build a CNF variable and save it in a config file.
-
- :param str kind: name of the CNF variable
- :param int instance: instance number of the CNF variable
- :param vals: tuples of enumerated action and subtuple with data
- :type vals: [(int, (str, int, str))]
- :param str data: data for the CNF variable
- :param filename: optional custom name of the config file
- :type filename: str or None
- :returns: name of the saved config file
- :rtype: str
- """
- builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
- batch_update_cnf(builder, vals)
- filename = generate_config_path(dumped=True) if filename is None else filename
- [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
- builder.save(filename)
- return filename
-
git://developer.intra2net.com / pyi2ncommon / blobdiff