1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 DEPRECATED! Please do not extend this or add new uses of this module, use
28 :py:mod:`pyi2ncommon.arnied_api` or :py:mod:`pyi2ncommon.cnfvar` instead!
30 Copyright: Intra2net AG
32 There are three types of setting some cnfvar configuration:
34 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
35 config file without modifying its content in any way
36 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
37 static file but rather as a template, replacing regex-matched values to
38 adapt it to different configurations
39 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
40 and custom cnfvar classes and writing them into config files of a desired
41 format (json, cnf, or raw)
45 ------------------------------------------------------
57 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
59 from .cnfline import build_cnfvar
60 from . import cnfvar_old
65 #: default set_cnf binary
66 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
67 #: default arnied_helper binary
68 BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
69 #: default location for template configuration files
71 #: default location for dumped configuration files
75 class ConfigError(Exception):
79 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
81 Universal command run wrapper.
83 :param str cmd: command to run
84 :param bool ignore_errors: whether not to raise error on command failure
85 :param vm: vm to run on if running on a guest instead of the host
86 :type vm: :py:class:`virttest.qemu_vm.VM` or None
87 :param int timeout: amount of seconds to wait for the program to run
88 :returns: command result output where output (stdout/stderr) is bytes
89 (encoding dependent on environment and command given)
90 :rtype: :py:class:`subprocess.CompletedProcess`
91 :raises: :py:class:`OSError` if command failed and cannot be ignored
94 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
95 stdout = stdout.encode()
100 if not ignore_errors:
101 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
102 return subprocess.CompletedProcess(cmd, status,
103 stdout=stdout, stderr=stderr)
105 return subprocess.run(cmd, check=not ignore_errors, shell=True,
109 def verify_running(process='arnied', timeout=60, vm=None):
111 Verify if a given process is running via 'pgrep'.
113 :param str process: process to verify if running
114 :param int timeout: run verification timeout
115 :param vm: vm to run on if running on a guest instead of the host
116 :type vm: :py:class:`virttest.qemu_vm.VM` or None
117 :raises: :py:class:`RuntimeError` if process is not running
122 platform_str = " on %s" % vm.name
123 for i in range(timeout):
124 log.info("Checking whether %s is running%s (%i\%i)",
125 process, platform_str, i, timeout)
126 result = run_cmd(cmd="pgrep -l -x %s" % process,
127 ignore_errors=True, vm=vm)
128 if result.returncode == 0:
132 raise RuntimeError("Process %s does not seem to be running" % process)
135 # Basic functionality
138 def accept_licence(vm=None):
140 Accept the Intra2net license.
142 :param vm: vm to run on if running on a guest instead of the host
143 :type vm: :py:class:`virttest.qemu_vm.VM` or None
145 This is mostly useful for simplified webpage access.
147 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
148 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
150 wait_for_generate(vm=vm)
153 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
155 Go online with the given provider id.
157 :param provider_id: provider to go online with
158 :type provider_id: int
159 :param wait_online: whether to wait until online
160 :type wait_online: bool
161 :param vm: vm to run on if running on a guest instead of the host
162 :type vm: :py:class:`virttest.qemu_vm.VM` or None
164 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
166 log.info("Switching to online mode with provider %d", provider_id)
168 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
169 if b'PROVIDER,' not in get_cnf_res.stdout:
170 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
174 cmd = 'tell-connd --online P%i' % provider_id
175 result = run_cmd(cmd=cmd, vm=vm)
179 wait_for_online(provider_id, timeout=timeout, vm=vm)
182 def go_offline(wait_offline=True, vm=None):
186 :param wait_offline: whether to wait until offline
187 :type wait_offline: bool
188 :param vm: vm to run on if running on a guest instead of the host
189 :type vm: :py:class:`virttest.qemu_vm.VM` or None
191 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
193 cmd = 'tell-connd --offline'
194 result = run_cmd(cmd=cmd, vm=vm)
198 if wait_offline is True:
199 wait_for_offline(vm=vm)
201 wait_for_offline(wait_offline, vm=vm)
204 def wait_for_offline(timeout=60, vm=None):
206 Wait for arnied to signal we are offline.
208 :param int timeout: maximum timeout for waiting
209 :param vm: vm to run on if running on a guest instead of the host
210 :type vm: :py:class:`virttest.qemu_vm.VM` or None
212 _wait_for_online_status('offline', None, timeout, vm)
215 def wait_for_online(provider_id, timeout=60, vm=None):
217 Wait for arnied to signal we are online.
219 :param provider_id: provider to go online with
220 :type provider_id: int
221 :param int timeout: maximum timeout for waiting
222 :param vm: vm to run on if running on a guest instead of the host
223 :type vm: :py:class:`virttest.qemu_vm.VM` or None
225 _wait_for_online_status('online', provider_id, timeout, vm)
228 def _wait_for_online_status(status, provider_id, timeout, vm):
229 # Don't use tell-connd --status here since the actual
230 # ONLINE signal to arnied is transmitted
231 # asynchronously via arnieclient_muxer.
233 if status == 'online':
234 expected_output = 'DEFAULT: 2'
235 set_status_func = lambda: go_online(provider_id, False, vm)
236 elif status == 'offline':
237 expected_output = 'DEFAULT: 0'
238 set_status_func = lambda: go_offline(False, vm)
240 raise ValueError('expect status "online" or "offline", not "{0}"!'
243 log.info("Waiting for arnied to be {0} within {1} seconds"
244 .format(status, timeout))
246 for i in range(timeout):
247 # arnied might invalidate the connd "connection barrier"
248 # after generate was running and switch to OFFLINE (race condition).
249 # -> tell arnied every ten seconds to go online again
250 if i % 10 == 0 and i != 0:
253 cmd = '/usr/intranator/bin/get_var ONLINE'
254 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
257 if expected_output in result.stdout.decode():
258 log.info("arnied is {0}. Continuing.".format(status))
263 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
264 .format(status, timeout))
267 def disable_virscan(vm=None):
269 Disable virscan that could block GENERATE and thus all configurations.
271 :param vm: vm to run on if running on a guest instead of the host
272 :type vm: :py:class:`virttest.qemu_vm.VM` or None
274 log.info("Disabling virus database update")
275 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
277 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
278 result = run_cmd(cmd=cmd, vm=vm)
281 # TODO: this intervention should be solved in later arnied_helper tool
282 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
283 result = run_cmd(cmd=cmd, vm=vm)
285 log.info("Virus database update disabled")
288 def email_transfer(vm=None):
290 Transfer all the emails using the guest tool arnied_helper.
292 :param vm: vm to run on if running on a guest instead of the host
293 :type vm: :py:class:`virttest.qemu_vm.VM` or None
295 cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
296 result = run_cmd(cmd=cmd, vm=vm)
300 def wait_for_email_transfer(timeout=300, vm=None):
302 Wait until the mail queue is empty and all emails are sent.
304 :param int timeout: email transfer timeout
305 :param vm: vm to run on if running on a guest instead of the host
306 :type vm: :py:class:`virttest.qemu_vm.VM` or None
308 for i in range(timeout):
310 # Retrigger mail queue in case something is deferred
311 # by an amavisd-new reconfiguration
312 run_cmd(cmd='postqueue -f', vm=vm)
313 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
315 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
316 log.debug('SMTP queue is empty')
319 log.warning('Timeout reached but SMTP queue still not empty after {} s'
323 def schedule(program, exec_time=0, optional_args="", vm=None):
325 Schedule a program to be executed at a given unix time stamp.
327 :param str program: program whose execution is scheduled
328 :param int exec_time: scheduled time of program's execution
329 :param str optional_args: optional command line arguments
330 :param vm: vm to run on if running on a guest instead of the host
331 :type vm: :py:class:`virttest.qemu_vm.VM` or None
333 log.info("Scheduling %s to be executed at %i", program, exec_time)
334 schedule_dir = "/var/intranator/schedule"
335 # clean previous schedules of the same program
336 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
337 for file_name in files:
338 if file_name.startswith(program.upper()):
339 log.debug("Removing previous scheduled %s", file_name)
341 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
343 os.unlink(os.path.join(schedule_dir, file_name))
345 contents = "%i\n%s\n" % (exec_time, optional_args)
347 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
348 prefix=program.upper() + "_",
351 log.debug("Created temporary file %s", tmp_file.name)
352 tmp_file.write(contents)
354 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
357 vm.copy_files_to(tmp_file.name, moved_tmp_file)
358 os.remove(tmp_file.name)
360 shutil.move(tmp_file.name, moved_tmp_file)
362 log.debug("Moved temporary file to %s", moved_tmp_file)
365 def wait_for_run(program, timeout=300, retries=10, vm=None):
367 Wait for a program using the guest arnied_helper tool.
369 :param str program: scheduled or running program to wait for
370 :param int timeout: program run timeout
371 :param int retries: number of tries to verify that the program is scheduled or running
372 :param vm: vm to run on if running on a guest instead of the host
373 :type vm: :py:class:`virttest.qemu_vm.VM` or None
375 log.info("Waiting for program %s to finish with timeout %i",
377 for i in range(retries):
378 cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
380 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
381 if check_scheduled.returncode == 0:
385 log.warning("The program %s was not scheduled and is not running", program)
387 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
388 f"{program.upper()} --wait-for-program-timeout {timeout}"
389 # add one second to make sure arnied_helper is finished when we expire
390 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
391 log.debug(result.stdout)
394 def wait_for_arnied(timeout=60, vm=None):
396 Wait for arnied socket to be ready.
398 :param int timeout: maximum number of seconds to wait
399 :param vm: vm to run on if running on a guest instead of the host
400 :type vm: :py:class:`virttest.qemu_vm.VM` or None
402 cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
403 f"--wait-for-arnied-socket-timeout {timeout}"
404 # add one second to make sure arnied_helper is finished when we expire
405 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
406 log.debug(result.stdout)
409 # Configuration functionality
411 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
413 Query arnied for a `cnf_key` and extract some information via regex.
415 :param str cnf_key: queried cnf key
416 :param int cnf_index: index of the cnf key
417 :param str regex: regex to apply on the queried cnf key data
418 :param bool compact: whether to retrieve compact version of the matched cnf keys
419 :param int timeout: arnied run verification timeout
420 :param vm: vm to run on if running on a guest instead of the host
421 :type vm: :py:class:`virttest.qemu_vm.VM` or None
422 :returns: extracted information via the regex
425 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
427 wait_for_arnied(timeout=timeout, vm=vm)
430 platform_str = " from %s" % vm.name
431 log.info("Extracting arnied value %s for %s%s using pattern %s",
432 cnf_index, cnf_key, platform_str, regex)
433 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
434 " %s" % cnf_index if cnf_index != -1 else "")
435 # get_cnf creates latin1-encoded output, transfer from VM removes non-ascii
436 output = run_cmd(cmd=cmd, vm=vm).stdout.decode('latin1')
437 return re.search(regex, output, flags=re.DOTALL)
440 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
442 Get the id of a configuration of type `cnf_key` and name `value`.
444 :param str cnf_key: queried cnf key
445 :param str value: cnf value of the cnf key
446 :param int timeout: arnied run verification timeout
447 :param vm: vm to run on if running on a guest instead of the host
448 :type vm: :py:class:`virttest.qemu_vm.VM` or None
449 :returns: the cnf id or -1 if no such cnf variable
452 wait_for_arnied(timeout=timeout, vm=vm)
453 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
454 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
458 cnf_id = int(cnf_id.group(1))
459 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
463 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
465 Invoke get_cnf and return a nested CNF structure.
467 :param str varname: "varname" field of the CNF_VAR to look up
468 :param instance: "instance" of that variable to return
470 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
471 :param int timeout: arnied run verification timeout
472 :param vm: vm to run on if running on a guest instead of the host
473 :type vm: :py:class:`virttest.qemu_vm.VM` or None
474 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
475 :rtype: cnfvar option
477 wait_for_arnied(timeout=timeout, vm=vm)
478 # firstly, build argv for get_cnf
479 cmd = ["get_cnf", "-j"]
480 if varname is not None:
481 cmd.append("%s" % varname)
483 cmd.append("%d" % instance)
484 cmd_line = " ".join(cmd)
487 result = run_cmd(cmd=cmd_line, vm=vm)
488 (status, raw) = result.returncode, result.stdout
490 log.info("error %d executing \"%s\"", status, cmd_line)
494 # reading was successful, attempt to parse what we got
496 # The output from "get_cnf -j" is already utf-8. This contrast with
497 # the output of "get_cnf" (no json) which is latin1.
498 if isinstance(raw, bytes):
499 raw = raw.decode("utf-8")
500 cnf = cnfvar_old.read_cnf_json(raw)
501 except TypeError as exn:
502 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
504 except cnfvar_old.InvalidCNF as exn:
505 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
509 return cnfvar_old.get_vars(cnf, data=data)
514 def get_cnfvar_id(varname, data, timeout=30, vm=None):
516 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
518 :param str varname: "varname" field of the CNF_VAR to look up
519 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
520 :param int timeout: arnied run verification timeout
521 :param vm: vm to run on if running on a guest instead of the host
522 :type vm: :py:class:`virttest.qemu_vm.VM` or None
523 :returns: the cnf id or -1 if no such cnf variable
526 wait_for_arnied(timeout=timeout, vm=vm)
527 log.info("Extracting from arnied CNF_VAR %s with data %s",
529 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
530 variables = cnf["cnf"]
531 if len(variables) == 0:
532 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
535 first_instance = int(variables[0]["instance"])
536 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
537 len(variables), first_instance)
538 return first_instance
541 def wait_for_generate(timeout=300, vm=None):
543 Wait for the 'generate' program to complete.
545 Arguments are similar to the ones from :py:method:`wait_for_run`.
547 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
548 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
551 def unset_cnf(varname="", instance="", timeout=30, vm=None):
553 Remove configuration from arnied.
555 :param str varname: "varname" field of the CNF_VAR to unset
556 :param int instance: "instance" of that variable to unset
557 :param int timeout: arnied run verification timeout
558 :param vm: vm to run on if running on a guest instead of the host
559 :type vm: :py:class:`virttest.qemu_vm.VM` or None
561 wait_for_arnied(timeout=timeout, vm=vm)
563 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
564 run_cmd(cmd=cmd, vm=vm)
566 wait_for_generate(vm=vm)
569 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
571 Perform static arnied configuration through a set of config files.
573 :param config_files: config files to use for the configuration
574 :type config_files: [str]
575 :param str kind: "json" or "cnf"
576 :param int timeout: arnied run verification timeout
577 :param vm: vm to run on if running on a guest instead of the host
578 :type vm: :py:class:`virttest.qemu_vm.VM` or None
579 :raises: :py:class:`ConfigError` if cannot apply file
581 The config files must be provided and are always expected to be found on
582 the host. If these are absolute paths, they will be kept as is or
583 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
584 the config files will be copied there as temporary files before applying.
586 log.info("Setting arnied configuration")
587 wait_for_arnied(timeout=timeout, vm=vm)
589 config_paths = prep_config_paths(config_files)
590 for config_path in config_paths:
591 with open(config_path, "rt", errors='replace') as config:
592 log.debug("Contents of applied %s:\n%s", config_path, config.read())
594 new_config_path = generate_config_path()
595 vm.copy_files_to(config_path, new_config_path)
596 config_path = new_config_path
597 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
599 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
600 logging.debug(result)
601 if result.returncode != 0:
602 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
604 " on %s" % vm.name if vm is not None else "",
608 wait_for_generate(vm=vm)
609 except Exception as ex:
610 # handle cases of remote configuration that leads to connection meltdown
611 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
612 log.info("Resetting connection to %s", vm.name)
613 vm.session = vm.wait_for_login(timeout=10)
614 log.debug("Connection reset via remote error: %s", ex)
619 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
620 kind="cnf", timeout=30, vm=None):
622 Perform semi-dynamic arnied configuration from an updated version of the
625 :param config_files: config files to use for the configuration
626 :type config_files: [str]
627 :param params_dict: parameters to override the defaults in the config files
628 :type params_dict: {str, str}
629 :param regex_dict: regular expressions to use for matching the overriden parameters
630 :type regex_dict: {str, str} or None
631 :param str kind: "json" or "cnf"
632 :param int timeout: arnied run verification timeout
633 :param vm: vm to run on if running on a guest instead of the host
634 :type vm: :py:class:`virttest.qemu_vm.VM` or None
636 The config files must be provided and are always expected to be found on
637 the host. If these are absolute paths, they will be kept as is or
638 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
639 the config files will be copied there as temporary files before applying.
641 log.info("Performing semi-dynamic arnied configuration")
643 config_paths = prep_cnf(config_files, params_dict, regex_dict)
644 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
646 log.info("Semi-dynamic arnied configuration successful!")
649 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
651 Perform dynamic arnied configuration from fully generated config files.
653 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
654 :type cnf: {str, str}
655 :param config_file: optional user supplied filename
656 :type config_file: str or None
657 :param str kind: "json", "cnf", or "raw"
658 :param int timeout: arnied run verification timeout
659 :param vm: vm to run on if running on a guest instead of the host
660 :type vm: :py:class:`virttest.qemu_vm.VM` or None
661 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
662 :raises: :py:class:`ConfigError` if cannot apply file
664 The config file might not be provided in which case a temporary file will
665 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
666 an absolute path. If a vm is provided, the config file will be copied there
667 as a temporary file before applying.
669 if config_file is None:
670 config_path = generate_config_path(dumped=True)
671 elif os.path.isabs(config_file):
672 config_path = config_file
674 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
675 generated = config_file is None
676 config_file = os.path.basename(config_path)
677 log.info("Using %s cnf file %s%s",
678 "generated" if generated else "user-supplied",
679 config_file, " on %s" % vm.name if vm is not None else "")
681 # Important to write bytes here to ensure text is encoded with latin-1
682 fd = open(config_path, "wb")
685 "raw": cnfvar_old.write_cnf_raw,
686 "json": cnfvar_old.write_cnf_json,
687 "cnf": cnfvar_old.write_cnf
689 SET_CNF_METHODS[kind](cnf, out=fd)
691 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
695 log.info("Generated config file %s", config_path)
697 kind = "cnf" if kind != "json" else kind
698 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
701 def set_cnf_pipe(cnf, timeout=30, block=False):
703 Set local configuration by talking to arnied via ``set_cnf``.
705 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
706 :type cnf: {str, str}
707 :param int timeout: arnied run verification timeout
708 :param bool block: whether to wait for generate to complete the
710 :returns: whether ``set_cnf`` succeeded or not
713 This is obviously not generic but supposed to be run on the guest.
715 log.info("Setting arnied configuration through local pipe")
716 wait_for_arnied(timeout=timeout)
718 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
721 log.error("Error applying configuration; status=%r" % exit)
722 log.error("and stderr:\n%s" % out)
724 log.debug("Configuration successfully passed to set_cnf, "
725 "read %d B from pipe" % len(out))
728 log.debug("Waiting for config job to complete")
731 log.debug("Exiting sucessfully")
735 def prep_config_paths(config_files, config_dir=None):
737 Prepare absolute paths for all configs at an expected location.
739 :param config_files: config files to use for the configuration
740 :type config_files: [str]
741 :param config_dir: config directory to prepend to the filepaths
742 :type config_dir: str or None
743 :returns: list of the full config paths
746 if config_dir is None:
747 config_dir = SRC_CONFIG_DIR
749 for config_file in config_files:
750 if os.path.isabs(config_file):
751 # Absolute path: The user requested a specific file
752 # f.e. needed for dynamic arnied config update
753 config_path = config_file
755 config_path = os.path.join(os.path.abspath(config_dir),
757 logging.debug("Using %s for original path %s", config_path, config_file)
758 config_paths.append(config_path)
762 def prep_cnf_value(config_file, value,
763 regex=None, template_key=None, ignore_fail=False):
765 Replace value in a provided arnied config file.
767 :param str config_file: file to use for the replacement
768 :param str value: value to replace the first matched group with
769 :param regex: regular expression to use when replacing a cnf value
770 :type regex: str or None
771 :param template_key: key of a quick template to use for the regex
772 :type template_key: str or None
773 :param bool ignore_fail: whether to ignore regex mismatching
774 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
776 In order to ensure better matching capabilities you are supposed to
777 provide a regex pattern with at least one subgroup to match your value.
778 What this means is that the value you like to replace is not directly
779 searched into the config text but matched within a larger regex in
780 in order to avoid any mismatch.
783 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
785 if template_key is None:
786 pattern = regex.encode()
788 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
789 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
790 pattern = samples[template_key].encode()
792 with open(config_file, "rb") as file_handle:
793 text = file_handle.read()
794 match_line = re.search(pattern, text)
796 if match_line is None and not ignore_fail:
797 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
798 elif match_line is not None:
799 old_line = match_line.group(0)
800 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
801 line = re.search(pattern, text).group(0)
802 log.debug("Updating %s to %s in %s", old_line, line, config_file)
803 with open(config_file, "wb") as file_handle:
804 file_handle.write(text)
807 def prep_cnf(config_files, params_dict, regex_dict=None):
809 Update all config files with the default overriding parameters,
810 i.e. override the values hard-coded in those config files.
812 :param config_files: config files to use for the configuration
813 :type config_files: [str]
814 :param params_dict: parameters to override the defaults in the config files
815 :type params_dict: {str, str}
816 :param regex_dict: regular expressions to use for matching the overriden parameters
817 :type regex_dict: {str, str} or None
818 :returns: list of prepared (modified) config paths
821 log.info("Preparing %s template config files", len(config_files))
823 src_config_paths = prep_config_paths(config_files)
824 new_config_paths = []
825 for config_path in src_config_paths:
826 new_config_path = generate_config_path(dumped=True)
827 shutil.copy(config_path, new_config_path)
828 new_config_paths.append(new_config_path)
830 for config_path in new_config_paths:
831 for param_key in params_dict.keys():
832 if regex_dict is None:
833 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
834 elif param_key in regex_dict.keys():
835 regex_val = regex_dict[param_key] % param_key.upper()
836 elif re.match("\w*_\d+$", param_key):
837 final_parameter, parent_id = \
838 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
839 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
840 % (parent_id, final_parameter.upper())
841 log.debug("Requested regex for %s is '%s'",
842 param_key, regex_val)
844 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
845 prep_cnf_value(config_path, params_dict[param_key],
846 regex=regex_val, ignore_fail=True)
847 log.info("Prepared template config file %s", config_path)
849 return new_config_paths
852 def generate_config_path(dumped=False):
854 Generate path for a temporary config name.
856 :param bool dumped: whether the file should be in the dump
857 directory or in temporary directory
858 :returns: generated config file path
861 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
862 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
875 def batch_update_cnf(cnf, vars):
877 Perform a batch update of multiple cnf variables.
879 :param cnf: CNF variable to update
880 :type cnf: BuildCnfVar object
881 :param vars: tuples of enumerated action and subtuple with data
882 :type vars: [(int, (str, int, str))]
883 :returns: updated CNF variable
884 :rtype: BuildCnfVar object
886 The actions are indexed in the same order: delete, update, add, child.
889 for (action, data) in vars:
892 last = cnf.update_cnf(var, ref, val)
895 last = cnf.add_cnf(var, ref, val)
896 elif action == Delete:
897 last = cnf.del_cnf(data)
898 elif action == Child: # only one depth supported
901 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
905 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
907 Build a CNF variable and save it in a config file.
909 :param str kind: name of the CNF variable
910 :param int instance: instance number of the CNF variable
911 :param vals: tuples of enumerated action and subtuple with data
912 :type vals: [(int, (str, int, str))]
913 :param str data: data for the CNF variable
914 :param filename: optional custom name of the config file
915 :type filename: str or None
916 :returns: name of the saved config file
919 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
920 batch_update_cnf(builder, vals)
921 filename = generate_config_path(dumped=True) if filename is None else filename
922 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
923 builder.save(filename)