1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 .. note:: Partially DEPRECATED! Use :py:mod:`pyi2ncommon.arnied_api` or
28 :py:mod:`pyi2ncommon.cnfvar` whenever possible. In particluar, do not
29 use or extend functionality regarding configuration (`get_cnf`,
32 Copyright: Intra2net AG
34 There are three types of setting some cnfvar configuration:
36 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
37 config file without modifying its content in any way
38 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
39 static file but rather as a template, replacing regex-matched values to
40 adapt it to different configurations
41 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
42 and custom cnfvar classes and writing them into config files of a desired
43 format (json, cnf, or raw)
47 ------------------------------------------------------
59 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
61 from .cnfline import build_cnfvar
62 from . import cnfvar_old
67 #: default set_cnf binary
68 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
69 #: default arnied_helper binary
70 BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
71 #: default location for template configuration files
73 #: default location for dumped configuration files
77 class ConfigError(Exception):
81 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
83 Universal command run wrapper.
85 :param str cmd: command to run
86 :param bool ignore_errors: whether not to raise error on command failure
87 :param vm: vm to run on if running on a guest instead of the host
88 :type vm: :py:class:`virttest.qemu_vm.VM` or None
89 :param int timeout: amount of seconds to wait for the program to run
90 :returns: command result output where output (stdout/stderr) is bytes
91 (encoding dependent on environment and command given)
92 :rtype: :py:class:`subprocess.CompletedProcess`
93 :raises: :py:class:`OSError` if command failed and cannot be ignored
96 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
97 stdout = stdout.encode()
102 if not ignore_errors:
103 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
104 return subprocess.CompletedProcess(cmd, status,
105 stdout=stdout, stderr=stderr)
107 return subprocess.run(cmd, check=not ignore_errors, shell=True,
111 def verify_running(process='arnied', timeout=60, vm=None):
113 Verify if a given process is running via 'pgrep'.
115 :param str process: process to verify if running
116 :param int timeout: run verification timeout
117 :param vm: vm to run on if running on a guest instead of the host
118 :type vm: :py:class:`virttest.qemu_vm.VM` or None
119 :raises: :py:class:`RuntimeError` if process is not running
124 platform_str = " on %s" % vm.name
125 for i in range(timeout):
126 log.info("Checking whether %s is running%s (%i\%i)",
127 process, platform_str, i, timeout)
128 result = run_cmd(cmd="pgrep -l -x %s" % process,
129 ignore_errors=True, vm=vm)
130 if result.returncode == 0:
134 raise RuntimeError("Process %s does not seem to be running" % process)
137 # Basic functionality
140 def accept_licence(vm=None):
142 Accept the Intra2net license.
144 :param vm: vm to run on if running on a guest instead of the host
145 :type vm: :py:class:`virttest.qemu_vm.VM` or None
147 This is mostly useful for simplified webpage access.
149 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
150 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
152 wait_for_generate(vm=vm)
155 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
157 Go online with the given provider id.
159 :param provider_id: provider to go online with
160 :type provider_id: int
161 :param wait_online: whether to wait until online
162 :type wait_online: bool
163 :param int timeout: Seconds to wait in :py:func:`wait_for_online`
164 :param vm: vm to run on if running on a guest instead of the host
165 :type vm: :py:class:`virttest.qemu_vm.VM` or None
167 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
169 log.info("Switching to online mode with provider %d", provider_id)
171 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
172 if b'PROVIDER,' not in get_cnf_res.stdout:
173 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
177 cmd = 'tell-connd --online P%i' % provider_id
178 result = run_cmd(cmd=cmd, vm=vm)
182 wait_for_online(provider_id, timeout=timeout, vm=vm)
185 def go_offline(wait_offline=True, vm=None):
189 :param wait_offline: whether to wait until offline
190 :type wait_offline: bool
191 :param vm: vm to run on if running on a guest instead of the host
192 :type vm: :py:class:`virttest.qemu_vm.VM` or None
194 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
196 cmd = 'tell-connd --offline'
197 result = run_cmd(cmd=cmd, vm=vm)
201 if wait_offline is True:
202 wait_for_offline(vm=vm)
204 wait_for_offline(wait_offline, vm=vm)
207 def wait_for_offline(timeout=60, vm=None):
209 Wait for arnied to signal we are offline.
211 :param int timeout: maximum timeout for waiting
212 :param vm: vm to run on if running on a guest instead of the host
213 :type vm: :py:class:`virttest.qemu_vm.VM` or None
215 _wait_for_online_status('offline', None, timeout, vm)
218 def wait_for_online(provider_id, timeout=60, vm=None):
220 Wait for arnied to signal we are online.
222 :param provider_id: provider to go online with
223 :type provider_id: int
224 :param int timeout: maximum timeout for waiting
225 :param vm: vm to run on if running on a guest instead of the host
226 :type vm: :py:class:`virttest.qemu_vm.VM` or None
228 _wait_for_online_status('online', provider_id, timeout, vm)
231 def _wait_for_online_status(status, provider_id, timeout, vm):
232 # Don't use tell-connd --status here since the actual
233 # ONLINE signal to arnied is transmitted
234 # asynchronously via arnieclient_muxer.
236 if status == 'online':
237 expected_output = 'DEFAULT: 2'
238 set_status_func = lambda: go_online(provider_id, False, vm)
239 elif status == 'offline':
240 expected_output = 'DEFAULT: 0'
241 set_status_func = lambda: go_offline(False, vm)
243 raise ValueError('expect status "online" or "offline", not "{0}"!'
246 log.info("Waiting for arnied to be {0} within {1} seconds"
247 .format(status, timeout))
249 for i in range(timeout):
250 # arnied might invalidate the connd "connection barrier"
251 # after generate was running and switch to OFFLINE (race condition).
252 # -> tell arnied every ten seconds to go online again
253 if i % 10 == 0 and i != 0:
256 cmd = '/usr/intranator/bin/get_var ONLINE'
257 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
260 if expected_output in result.stdout.decode():
261 log.info("arnied is {0}. Continuing.".format(status))
266 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
267 .format(status, timeout))
270 def disable_virscan(vm=None):
272 Disable virscan that could block GENERATE and thus all configurations.
274 :param vm: vm to run on if running on a guest instead of the host
275 :type vm: :py:class:`virttest.qemu_vm.VM` or None
277 log.info("Disabling virus database update")
278 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
280 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
281 result = run_cmd(cmd=cmd, vm=vm)
284 # TODO: this intervention should be solved in later arnied_helper tool
285 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
286 result = run_cmd(cmd=cmd, vm=vm)
288 log.info("Virus database update disabled")
291 def email_transfer(vm=None):
293 Transfer all the emails using the guest tool arnied_helper.
295 :param vm: vm to run on if running on a guest instead of the host
296 :type vm: :py:class:`virttest.qemu_vm.VM` or None
298 cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
299 result = run_cmd(cmd=cmd, vm=vm)
303 def wait_for_email_transfer(timeout=300, vm=None):
305 Wait until the mail queue is empty and all emails are sent.
307 :param int timeout: email transfer timeout
308 :param vm: vm to run on if running on a guest instead of the host
309 :type vm: :py:class:`virttest.qemu_vm.VM` or None
311 for i in range(timeout):
313 # Retrigger mail queue in case something is deferred
314 # by an amavisd-new reconfiguration
315 run_cmd(cmd='postqueue -f', vm=vm)
316 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
318 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
319 log.debug('SMTP queue is empty')
322 log.warning('Timeout reached but SMTP queue still not empty after {} s'
326 def schedule(program, exec_time=0, optional_args="", vm=None):
328 Schedule a program to be executed at a given unix time stamp.
330 :param str program: program whose execution is scheduled
331 :param int exec_time: scheduled time of program's execution
332 :param str optional_args: optional command line arguments
333 :param vm: vm to run on if running on a guest instead of the host
334 :type vm: :py:class:`virttest.qemu_vm.VM` or None
336 log.info("Scheduling %s to be executed at %i", program, exec_time)
337 schedule_dir = "/var/intranator/schedule"
338 # clean previous schedules of the same program
339 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
340 for file_name in files:
341 if file_name.startswith(program.upper()):
342 log.debug("Removing previous scheduled %s", file_name)
344 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
346 os.unlink(os.path.join(schedule_dir, file_name))
348 contents = "%i\n%s\n" % (exec_time, optional_args)
350 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
351 prefix=program.upper() + "_",
354 log.debug("Created temporary file %s", tmp_file.name)
355 tmp_file.write(contents)
357 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
360 vm.copy_files_to(tmp_file.name, moved_tmp_file)
361 os.remove(tmp_file.name)
363 shutil.move(tmp_file.name, moved_tmp_file)
365 log.debug("Moved temporary file to %s", moved_tmp_file)
368 def wait_for_run(program, timeout=300, retries=10, vm=None):
370 Wait for a program using the guest arnied_helper tool.
372 :param str program: scheduled or running program to wait for
373 :param int timeout: program run timeout
374 :param int retries: number of tries to verify that the program is scheduled or running
375 :param vm: vm to run on if running on a guest instead of the host
376 :type vm: :py:class:`virttest.qemu_vm.VM` or None
378 log.info("Waiting for program %s to finish with timeout %i",
380 for i in range(retries):
381 cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
383 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
384 if check_scheduled.returncode == 0:
388 log.warning("The program %s was not scheduled and is not running", program)
390 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
391 f"{program.upper()} --wait-for-program-timeout {timeout}"
392 # add one second to make sure arnied_helper is finished when we expire
393 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
394 log.debug(result.stdout)
397 def wait_for_arnied(timeout=60, vm=None):
399 Wait for arnied socket to be ready.
401 :param int timeout: maximum number of seconds to wait
402 :param vm: vm to run on if running on a guest instead of the host
403 :type vm: :py:class:`virttest.qemu_vm.VM` or None
405 cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
406 f"--wait-for-arnied-socket-timeout {timeout}"
407 # add one second to make sure arnied_helper is finished when we expire
408 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
409 log.debug(result.stdout)
412 # Configuration functionality
414 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
416 Query arnied for a `cnf_key` and extract some information via regex.
418 :param str cnf_key: queried cnf key
419 :param int cnf_index: index of the cnf key
420 :param str regex: regex to apply on the queried cnf key data
421 :param bool compact: whether to retrieve compact version of the matched cnf keys
422 :param int timeout: arnied run verification timeout
423 :param vm: vm to run on if running on a guest instead of the host
424 :type vm: :py:class:`virttest.qemu_vm.VM` or None
425 :returns: extracted information via the regex
428 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
430 wait_for_arnied(timeout=timeout, vm=vm)
433 platform_str = " from %s" % vm.name
434 log.info("Extracting arnied value %s for %s%s using pattern %s",
435 cnf_index, cnf_key, platform_str, regex)
436 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
437 " %s" % cnf_index if cnf_index != -1 else "")
438 # get_cnf creates latin1-encoded output, transfer from VM removes non-ascii
439 output = run_cmd(cmd=cmd, vm=vm).stdout.decode('latin1')
440 return re.search(regex, output, flags=re.DOTALL)
443 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
445 Get the id of a configuration of type `cnf_key` and name `value`.
447 :param str cnf_key: queried cnf key
448 :param str value: cnf value of the cnf key
449 :param int timeout: arnied run verification timeout
450 :param vm: vm to run on if running on a guest instead of the host
451 :type vm: :py:class:`virttest.qemu_vm.VM` or None
452 :returns: the cnf id or -1 if no such cnf variable
455 wait_for_arnied(timeout=timeout, vm=vm)
456 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
457 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
461 cnf_id = int(cnf_id.group(1))
462 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
466 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
468 Invoke get_cnf and return a nested CNF structure.
470 :param str varname: "varname" field of the CNF_VAR to look up
471 :param instance: "instance" of that variable to return
473 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
474 :param int timeout: arnied run verification timeout
475 :param vm: vm to run on if running on a guest instead of the host
476 :type vm: :py:class:`virttest.qemu_vm.VM` or None
477 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
478 :rtype: cnfvar option
480 wait_for_arnied(timeout=timeout, vm=vm)
481 # firstly, build argv for get_cnf
482 cmd = ["get_cnf", "-j"]
483 if varname is not None:
484 cmd.append("%s" % varname)
486 cmd.append("%d" % instance)
487 cmd_line = " ".join(cmd)
490 result = run_cmd(cmd=cmd_line, vm=vm)
491 (status, raw) = result.returncode, result.stdout
493 log.info("error %d executing \"%s\"", status, cmd_line)
497 # reading was successful, attempt to parse what we got
499 # The output from "get_cnf -j" is already utf-8. This contrast with
500 # the output of "get_cnf" (no json) which is latin1.
501 if isinstance(raw, bytes):
502 raw = raw.decode("utf-8")
503 cnf = cnfvar_old.read_cnf_json(raw)
504 except TypeError as exn:
505 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
507 except cnfvar_old.InvalidCNF as exn:
508 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
512 return cnfvar_old.get_vars(cnf, data=data)
517 def get_cnfvar_id(varname, data, timeout=30, vm=None):
519 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
521 :param str varname: "varname" field of the CNF_VAR to look up
522 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
523 :param int timeout: arnied run verification timeout
524 :param vm: vm to run on if running on a guest instead of the host
525 :type vm: :py:class:`virttest.qemu_vm.VM` or None
526 :returns: the cnf id or -1 if no such cnf variable
529 wait_for_arnied(timeout=timeout, vm=vm)
530 log.info("Extracting from arnied CNF_VAR %s with data %s",
532 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
533 variables = cnf["cnf"]
534 if len(variables) == 0:
535 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
538 first_instance = int(variables[0]["instance"])
539 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
540 len(variables), first_instance)
541 return first_instance
544 def wait_for_generate(timeout=300, vm=None):
546 Wait for the 'generate' program to complete.
548 Arguments are similar to the ones from :py:method:`wait_for_run`.
550 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
551 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
554 def unset_cnf(varname="", instance="", timeout=30, vm=None):
556 Remove configuration from arnied.
558 :param str varname: "varname" field of the CNF_VAR to unset
559 :param int instance: "instance" of that variable to unset
560 :param int timeout: arnied run verification timeout
561 :param vm: vm to run on if running on a guest instead of the host
562 :type vm: :py:class:`virttest.qemu_vm.VM` or None
564 wait_for_arnied(timeout=timeout, vm=vm)
566 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
567 run_cmd(cmd=cmd, vm=vm)
569 wait_for_generate(vm=vm)
572 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
574 Perform static arnied configuration through a set of config files.
576 :param config_files: config files to use for the configuration
577 :type config_files: [str]
578 :param str kind: "json" or "cnf"
579 :param int timeout: arnied run verification timeout
580 :param vm: vm to run on if running on a guest instead of the host
581 :type vm: :py:class:`virttest.qemu_vm.VM` or None
582 :raises: :py:class:`ConfigError` if cannot apply file
584 The config files must be provided and are always expected to be found on
585 the host. If these are absolute paths, they will be kept as is or
586 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
587 the config files will be copied there as temporary files before applying.
589 log.info("Setting arnied configuration")
590 wait_for_arnied(timeout=timeout, vm=vm)
592 config_paths = prep_config_paths(config_files)
593 for config_path in config_paths:
594 with open(config_path, "rt", errors='replace') as config:
595 log.debug("Contents of applied %s:\n%s", config_path, config.read())
597 new_config_path = generate_config_path()
598 vm.copy_files_to(config_path, new_config_path)
599 config_path = new_config_path
600 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
602 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
603 logging.debug(result)
604 if result.returncode != 0:
605 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
607 " on %s" % vm.name if vm is not None else "",
611 wait_for_generate(vm=vm)
612 except Exception as ex:
613 # handle cases of remote configuration that leads to connection meltdown
614 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
615 log.info("Resetting connection to %s", vm.name)
616 vm.session = vm.wait_for_login(timeout=10)
617 log.debug("Connection reset via remote error: %s", ex)
622 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
623 kind="cnf", timeout=30, vm=None):
625 Perform semi-dynamic arnied configuration from an updated version of the
628 :param config_files: config files to use for the configuration
629 :type config_files: [str]
630 :param params_dict: parameters to override the defaults in the config files
631 :type params_dict: {str, str}
632 :param regex_dict: regular expressions to use for matching the overriden parameters
633 :type regex_dict: {str, str} or None
634 :param str kind: "json" or "cnf"
635 :param int timeout: arnied run verification timeout
636 :param vm: vm to run on if running on a guest instead of the host
637 :type vm: :py:class:`virttest.qemu_vm.VM` or None
639 The config files must be provided and are always expected to be found on
640 the host. If these are absolute paths, they will be kept as is or
641 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
642 the config files will be copied there as temporary files before applying.
644 log.info("Performing semi-dynamic arnied configuration")
646 config_paths = prep_cnf(config_files, params_dict, regex_dict)
647 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
649 log.info("Semi-dynamic arnied configuration successful!")
652 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
654 Perform dynamic arnied configuration from fully generated config files.
656 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
657 :type cnf: {str, str}
658 :param config_file: optional user supplied filename
659 :type config_file: str or None
660 :param str kind: "json", "cnf", or "raw"
661 :param int timeout: arnied run verification timeout
662 :param vm: vm to run on if running on a guest instead of the host
663 :type vm: :py:class:`virttest.qemu_vm.VM` or None
664 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
665 :raises: :py:class:`ConfigError` if cannot apply file
667 The config file might not be provided in which case a temporary file will
668 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
669 an absolute path. If a vm is provided, the config file will be copied there
670 as a temporary file before applying.
672 if config_file is None:
673 config_path = generate_config_path(dumped=True)
674 elif os.path.isabs(config_file):
675 config_path = config_file
677 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
678 generated = config_file is None
679 config_file = os.path.basename(config_path)
680 log.info("Using %s cnf file %s%s",
681 "generated" if generated else "user-supplied",
682 config_file, " on %s" % vm.name if vm is not None else "")
684 # Important to write bytes here to ensure text is encoded with latin-1
685 fd = open(config_path, "wb")
688 "raw": cnfvar_old.write_cnf_raw,
689 "json": cnfvar_old.write_cnf_json,
690 "cnf": cnfvar_old.write_cnf
692 SET_CNF_METHODS[kind](cnf, out=fd)
694 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
698 log.info("Generated config file %s", config_path)
700 kind = "cnf" if kind != "json" else kind
701 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
704 def set_cnf_pipe(cnf, timeout=30, block=False):
706 Set local configuration by talking to arnied via ``set_cnf``.
708 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
709 :type cnf: {str, str}
710 :param int timeout: arnied run verification timeout
711 :param bool block: whether to wait for generate to complete the
713 :returns: whether ``set_cnf`` succeeded or not
716 This is obviously not generic but supposed to be run on the guest.
718 log.info("Setting arnied configuration through local pipe")
719 wait_for_arnied(timeout=timeout)
721 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
724 log.error("Error applying configuration; status=%r" % exit)
725 log.error("and stderr:\n%s" % out)
727 log.debug("Configuration successfully passed to set_cnf, "
728 "read %d B from pipe" % len(out))
731 log.debug("Waiting for config job to complete")
734 log.debug("Exiting sucessfully")
738 def prep_config_paths(config_files, config_dir=None):
740 Prepare absolute paths for all configs at an expected location.
742 :param config_files: config files to use for the configuration
743 :type config_files: [str]
744 :param config_dir: config directory to prepend to the filepaths
745 :type config_dir: str or None
746 :returns: list of the full config paths
749 if config_dir is None:
750 config_dir = SRC_CONFIG_DIR
752 for config_file in config_files:
753 if os.path.isabs(config_file):
754 # Absolute path: The user requested a specific file
755 # f.e. needed for dynamic arnied config update
756 config_path = config_file
758 config_path = os.path.join(os.path.abspath(config_dir),
760 logging.debug("Using %s for original path %s", config_path, config_file)
761 config_paths.append(config_path)
765 def prep_cnf_value(config_file, value,
766 regex=None, template_key=None, ignore_fail=False):
768 Replace value in a provided arnied config file.
770 :param str config_file: file to use for the replacement
771 :param str value: value to replace the first matched group with
772 :param regex: regular expression to use when replacing a cnf value
773 :type regex: str or None
774 :param template_key: key of a quick template to use for the regex
775 :type template_key: str or None
776 :param bool ignore_fail: whether to ignore regex mismatching
777 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
779 In order to ensure better matching capabilities you are supposed to
780 provide a regex pattern with at least one subgroup to match your value.
781 What this means is that the value you like to replace is not directly
782 searched into the config text but matched within a larger regex in
783 in order to avoid any mismatch.
786 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
788 if template_key is None:
789 pattern = regex.encode()
791 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
792 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
793 pattern = samples[template_key].encode()
795 with open(config_file, "rb") as file_handle:
796 text = file_handle.read()
797 match_line = re.search(pattern, text)
799 if match_line is None and not ignore_fail:
800 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
801 elif match_line is not None:
802 old_line = match_line.group(0)
803 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
804 line = re.search(pattern, text).group(0)
805 log.debug("Updating %s to %s in %s", old_line, line, config_file)
806 with open(config_file, "wb") as file_handle:
807 file_handle.write(text)
810 def prep_cnf(config_files, params_dict, regex_dict=None):
812 Update all config files with the default overriding parameters,
813 i.e. override the values hard-coded in those config files.
815 :param config_files: config files to use for the configuration
816 :type config_files: [str]
817 :param params_dict: parameters to override the defaults in the config files
818 :type params_dict: {str, str}
819 :param regex_dict: regular expressions to use for matching the overriden parameters
820 :type regex_dict: {str, str} or None
821 :returns: list of prepared (modified) config paths
824 log.info("Preparing %s template config files", len(config_files))
826 src_config_paths = prep_config_paths(config_files)
827 new_config_paths = []
828 for config_path in src_config_paths:
829 new_config_path = generate_config_path(dumped=True)
830 shutil.copy(config_path, new_config_path)
831 new_config_paths.append(new_config_path)
833 for config_path in new_config_paths:
834 for param_key in params_dict.keys():
835 if regex_dict is None:
836 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
837 elif param_key in regex_dict.keys():
838 regex_val = regex_dict[param_key] % param_key.upper()
839 elif re.match("\w*_\d+$", param_key):
840 final_parameter, parent_id = \
841 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
842 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
843 % (parent_id, final_parameter.upper())
844 log.debug("Requested regex for %s is '%s'",
845 param_key, regex_val)
847 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
848 prep_cnf_value(config_path, params_dict[param_key],
849 regex=regex_val, ignore_fail=True)
850 log.info("Prepared template config file %s", config_path)
852 return new_config_paths
855 def generate_config_path(dumped=False):
857 Generate path for a temporary config name.
859 :param bool dumped: whether the file should be in the dump
860 directory or in temporary directory
861 :returns: generated config file path
864 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
865 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
878 def batch_update_cnf(cnf, vars):
880 Perform a batch update of multiple cnf variables.
882 :param cnf: CNF variable to update
883 :type cnf: BuildCnfVar object
884 :param vars: tuples of enumerated action and subtuple with data
885 :type vars: [(int, (str, int, str))]
886 :returns: updated CNF variable
887 :rtype: BuildCnfVar object
889 The actions are indexed in the same order: delete, update, add, child.
892 for (action, data) in vars:
895 last = cnf.update_cnf(var, ref, val)
898 last = cnf.add_cnf(var, ref, val)
899 elif action == Delete:
900 last = cnf.del_cnf(data)
901 elif action == Child: # only one depth supported
904 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
908 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
910 Build a CNF variable and save it in a config file.
912 :param str kind: name of the CNF variable
913 :param int instance: instance number of the CNF variable
914 :param vals: tuples of enumerated action and subtuple with data
915 :type vals: [(int, (str, int, str))]
916 :param str data: data for the CNF variable
917 :param filename: optional custom name of the config file
918 :type filename: str or None
919 :returns: name of the saved config file
922 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
923 batch_update_cnf(builder, vals)
924 filename = generate_config_path(dumped=True) if filename is None else filename
925 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
926 builder.save(filename)