1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 Copyright: Intra2net AG
30 There are three types of setting some cnfvar configuration:
32 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
33 config file without modifying its content in any way
34 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
35 static file but rather as a template, replacing regex-matched values to
36 adapt it to different configurations
37 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
38 and custom cnfvar classes and writing them into config files of a desired
39 format (json, cnf, or raw)
43 ------------------------------------------------------
55 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
57 from .cnfline import build_cnfvar
63 #: default set_cnf binary
64 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
65 #: default arnied_helper binary
66 BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
67 #: default location for template configuration files
69 #: default location for dumped configuration files
73 class ConfigError(Exception):
77 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
79 Universal command run wrapper.
81 :param str cmd: command to run
82 :param bool ignore_errors: whether not to raise error on command failure
83 :param vm: vm to run on if running on a guest instead of the host
84 :type vm: :py:class:`virttest.qemu_vm.VM` or None
85 :param int timeout: amount of seconds to wait for the program to run
86 :returns: command result output
88 :raises: :py:class:`OSError` if command failed and cannot be ignored
91 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
92 stdout = stdout.encode()
98 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
99 return subprocess.CompletedProcess(cmd, status, stdout=stdout, stderr=stderr)
101 return subprocess.run(cmd, check=not ignore_errors, shell=True, capture_output=True)
104 def verify_running(process='arnied', timeout=60, vm=None):
106 Verify if a given process is running via 'pgrep'.
108 :param str process: process to verify if running
109 :param int timeout: run verification timeout
110 :param vm: vm to run on if running on a guest instead of the host
111 :type vm: :py:class:`virttest.qemu_vm.VM` or None
112 :raises: :py:class:`RuntimeError` if process is not running
117 platform_str = " on %s" % vm.name
118 for i in range(timeout):
119 log.info("Checking whether %s is running%s (%i\%i)",
120 process, platform_str, i, timeout)
121 result = run_cmd(cmd="pgrep -l -x %s" % process,
122 ignore_errors=True, vm=vm)
123 if result.returncode == 0:
127 raise RuntimeError("Process %s does not seem to be running" % process)
130 # Basic functionality
133 def accept_licence(vm=None):
135 Accept the Intra2net license.
137 :param vm: vm to run on if running on a guest instead of the host
138 :type vm: :py:class:`virttest.qemu_vm.VM` or None
140 This is mostly useful for simplified webpage access.
142 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
143 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
145 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end GENERATE"
146 run_cmd(cmd=cmd, vm=vm)
149 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
151 Go online with the given provider id.
153 :param provider_id: provider to go online with
154 :type provider_id: int
155 :param wait_online: whether to wait until online
156 :type wait_online: bool
157 :param vm: vm to run on if running on a guest instead of the host
158 :type vm: :py:class:`virttest.qemu_vm.VM` or None
160 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
162 log.info("Switching to online mode with provider %d", provider_id)
164 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
165 if b'PROVIDER,' not in get_cnf_res.stdout:
166 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
170 cmd = 'tell-connd --online P%i' % provider_id
171 result = run_cmd(cmd=cmd, vm=vm)
175 wait_for_online(provider_id, timeout=timeout, vm=vm)
178 def go_offline(wait_offline=True, vm=None):
182 :param wait_offline: whether to wait until offline
183 :type wait_offline: bool
184 :param vm: vm to run on if running on a guest instead of the host
185 :type vm: :py:class:`virttest.qemu_vm.VM` or None
187 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
189 cmd = 'tell-connd --offline'
190 result = run_cmd(cmd=cmd, vm=vm)
194 if wait_offline is True:
195 wait_for_offline(vm=vm)
197 wait_for_offline(wait_offline, vm=vm)
200 def wait_for_offline(timeout=60, vm=None):
202 Wait for arnied to signal we are offline.
204 :param int timeout: maximum timeout for waiting
205 :param vm: vm to run on if running on a guest instead of the host
206 :type vm: :py:class:`virttest.qemu_vm.VM` or None
208 _wait_for_online_status('offline', None, timeout, vm)
211 def wait_for_online(provider_id, timeout=60, vm=None):
213 Wait for arnied to signal we are online.
215 :param provider_id: provider to go online with
216 :type provider_id: int
217 :param int timeout: maximum timeout for waiting
218 :param vm: vm to run on if running on a guest instead of the host
219 :type vm: :py:class:`virttest.qemu_vm.VM` or None
221 _wait_for_online_status('online', provider_id, timeout, vm)
224 def _wait_for_online_status(status, provider_id, timeout, vm):
225 # Don't use tell-connd --status here since the actual
226 # ONLINE signal to arnied is transmitted
227 # asynchronously via arnieclient_muxer.
229 if status == 'online':
230 expected_output = 'DEFAULT: 2'
231 set_status_func = lambda: go_online(provider_id, False, vm)
232 elif status == 'offline':
233 expected_output = 'DEFAULT: 0'
234 set_status_func = lambda: go_offline(False, vm)
236 raise ValueError('expect status "online" or "offline", not "{0}"!'
239 log.info("Waiting for arnied to be {0} within {1} seconds"
240 .format(status, timeout))
242 for i in range(timeout):
243 # arnied might invalidate the connd "connection barrier"
244 # after generate was running and switch to OFFLINE (race condition).
245 # -> tell arnied every ten seconds to go online again
246 if i % 10 == 0 and i != 0:
249 cmd = '/usr/intranator/bin/get_var ONLINE'
250 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
253 if expected_output in result.stdout.decode():
254 log.info("arnied is {0}. Continuing.".format(status))
259 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
260 .format(status, timeout))
263 def disable_virscan(vm=None):
265 Disable virscan that could block GENERATE and thus all configurations.
267 :param vm: vm to run on if running on a guest instead of the host
268 :type vm: :py:class:`virttest.qemu_vm.VM` or None
270 log.info("Disabling virus database update")
271 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
273 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
274 result = run_cmd(cmd=cmd, vm=vm)
277 # TODO: this intervention should be solved in later arnied_helper tool
278 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
279 result = run_cmd(cmd=cmd, vm=vm)
281 log.info("Virus database update disabled")
284 def email_transfer(vm=None):
286 Transfer all the emails using the guest tool arnied_helper.
288 :param vm: vm to run on if running on a guest instead of the host
289 :type vm: :py:class:`virttest.qemu_vm.VM` or None
291 cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
292 result = run_cmd(cmd=cmd, vm=vm)
296 def wait_for_email_transfer(timeout=300, vm=None):
298 Wait until the mail queue is empty and all emails are sent.
300 :param int timeout: email transfer timeout
301 :param vm: vm to run on if running on a guest instead of the host
302 :type vm: :py:class:`virttest.qemu_vm.VM` or None
304 for i in range(timeout):
306 # Retrigger mail queue in case something is deferred
307 # by an amavisd-new reconfiguration
308 run_cmd(cmd='postqueue -f', vm=vm)
309 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
311 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
312 log.debug('SMTP queue is empty')
315 log.warning('Timeout reached but SMTP queue still not empty after {} s'
319 def schedule(program, exec_time=0, optional_args="", vm=None):
321 Schedule a program to be executed at a given unix time stamp.
323 :param str program: program whose execution is scheduled
324 :param int exec_time: scheduled time of program's execution
325 :param str optional_args: optional command line arguments
326 :param vm: vm to run on if running on a guest instead of the host
327 :type vm: :py:class:`virttest.qemu_vm.VM` or None
329 log.info("Scheduling %s to be executed at %i", program, exec_time)
330 schedule_dir = "/var/intranator/schedule"
331 # clean previous schedules of the same program
332 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
333 for file_name in files:
334 if file_name.startswith(program.upper()):
335 log.debug("Removing previous scheduled %s", file_name)
337 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
339 os.unlink(os.path.join(schedule_dir, file_name))
341 contents = "%i\n%s\n" % (exec_time, optional_args)
343 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
344 prefix=program.upper() + "_",
347 log.debug("Created temporary file %s", tmp_file.name)
348 tmp_file.write(contents)
350 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
353 vm.copy_files_to(tmp_file.name, moved_tmp_file)
354 os.remove(tmp_file.name)
356 shutil.move(tmp_file.name, moved_tmp_file)
358 log.debug("Moved temporary file to %s", moved_tmp_file)
361 def wait_for_run(program, timeout=300, retries=10, vm=None):
363 Wait for a program using the guest arnied_helper tool.
365 :param str program: scheduled or running program to wait for
366 :param int timeout: program run timeout
367 :param int retries: number of tries to verify that the program is scheduled or running
368 :param vm: vm to run on if running on a guest instead of the host
369 :type vm: :py:class:`virttest.qemu_vm.VM` or None
371 log.info("Waiting for program %s to finish with timeout %i",
373 for i in range(retries):
374 cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
376 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
377 if check_scheduled.returncode == 0:
381 log.warning("The program %s was not scheduled and is not running", program)
383 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
384 f"{program.upper()} --wait-for-program-timeout {timeout}"
385 # add one second to make sure arnied_helper is finished when we expire
386 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
387 log.debug(result.stdout)
390 def wait_for_arnied(timeout=60, vm=None):
392 Wait for arnied socket to be ready.
394 :param int timeout: maximum number of seconds to wait
395 :param vm: vm to run on if running on a guest instead of the host
396 :type vm: :py:class:`virttest.qemu_vm.VM` or None
398 cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
399 f"--wait-for-arnied-socket-timeout {timeout}"
400 # add one second to make sure arnied_helper is finished when we expire
401 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
402 log.debug(result.stdout)
405 # Configuration functionality
407 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
409 Query arnied for a `cnf_key` and extract some information via regex.
411 :param str cnf_key: queried cnf key
412 :param int cnf_index: index of the cnf key
413 :param str regex: regex to apply on the queried cnf key data
414 :param bool compact: whether to retrieve compact version of the matched cnf keys
415 :param int timeout: arnied run verification timeout
416 :param vm: vm to run on if running on a guest instead of the host
417 :type vm: :py:class:`virttest.qemu_vm.VM` or None
418 :returns: extracted information via the regex
421 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
423 wait_for_arnied(timeout=timeout, vm=vm)
426 platform_str = " from %s" % vm.name
427 log.info("Extracting arnied value %s for %s%s using pattern %s",
428 cnf_index, cnf_key, platform_str, regex)
429 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
430 " %s" % cnf_index if cnf_index != -1 else "")
431 output = run_cmd(cmd=cmd, vm=vm).stdout.decode()
432 return re.search(regex, output, flags=re.DOTALL)
435 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
437 Get the id of a configuration of type `cnf_key` and name `value`.
439 :param str cnf_key: queried cnf key
440 :param str value: cnf value of the cnf key
441 :param int timeout: arnied run verification timeout
442 :param vm: vm to run on if running on a guest instead of the host
443 :type vm: :py:class:`virttest.qemu_vm.VM` or None
444 :returns: the cnf id or -1 if no such cnf variable
447 wait_for_arnied(timeout=timeout, vm=vm)
448 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
449 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
453 cnf_id = int(cnf_id.group(1))
454 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
458 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
460 Invoke get_cnf and return a nested CNF structure.
462 :param str varname: "varname" field of the CNF_VAR to look up
463 :param instance: "instance" of that variable to return
465 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
466 :param int timeout: arnied run verification timeout
467 :param vm: vm to run on if running on a guest instead of the host
468 :type vm: :py:class:`virttest.qemu_vm.VM` or None
469 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
470 :rtype: cnfvar option
472 wait_for_arnied(timeout=timeout, vm=vm)
473 # firstly, build argv for get_cnf
474 cmd = ["get_cnf", "-j"]
475 if varname is not None:
476 cmd.append("%s" % varname)
478 cmd.append("%d" % instance)
479 cmd_line = " ".join(cmd)
482 result = run_cmd(cmd=cmd_line, vm=vm)
483 (status, raw) = result.returncode, result.stdout
485 log.info("error %d executing \"%s\"", status, cmd_line)
489 # reading was successful, attempt to parse what we got
491 # The output from "get_cnf -j" is already utf-8. This contrast with
492 # the output of "get_cnf" (no json) which is latin1.
493 if isinstance(raw, bytes):
494 raw = raw.decode("utf-8")
495 cnf = cnfvar.read_cnf_json(raw)
496 except TypeError as exn:
497 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
499 except cnfvar.InvalidCNF as exn:
500 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
504 return cnfvar.get_vars(cnf, data=data)
509 def get_cnfvar_id(varname, data, timeout=30, vm=None):
511 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
513 :param str varname: "varname" field of the CNF_VAR to look up
514 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
515 :param int timeout: arnied run verification timeout
516 :param vm: vm to run on if running on a guest instead of the host
517 :type vm: :py:class:`virttest.qemu_vm.VM` or None
518 :returns: the cnf id or -1 if no such cnf variable
521 wait_for_arnied(timeout=timeout, vm=vm)
522 log.info("Extracting from arnied CNF_VAR %s with data %s",
524 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
525 variables = cnf["cnf"]
526 if len(variables) == 0:
527 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
530 first_instance = int(variables[0]["instance"])
531 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
532 len(variables), first_instance)
533 return first_instance
536 def wait_for_generate(timeout=300, vm=None):
538 Wait for the 'generate' program to complete.
540 Arguments are similar to the ones from :py:method:`wait_for_run`.
542 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
543 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
546 def unset_cnf(varname="", instance="", timeout=30, vm=None):
548 Remove configuration from arnied.
550 :param str varname: "varname" field of the CNF_VAR to unset
551 :param int instance: "instance" of that variable to unset
552 :param int timeout: arnied run verification timeout
553 :param vm: vm to run on if running on a guest instead of the host
554 :type vm: :py:class:`virttest.qemu_vm.VM` or None
556 wait_for_arnied(timeout=timeout, vm=vm)
558 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
559 run_cmd(cmd=cmd, vm=vm)
561 wait_for_generate(vm=vm)
564 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
566 Perform static arnied configuration through a set of config files.
568 :param config_files: config files to use for the configuration
569 :type config_files: [str]
570 :param str kind: "json" or "cnf"
571 :param int timeout: arnied run verification timeout
572 :param vm: vm to run on if running on a guest instead of the host
573 :type vm: :py:class:`virttest.qemu_vm.VM` or None
574 :raises: :py:class:`ConfigError` if cannot apply file
576 The config files must be provided and are always expected to be found on
577 the host. If these are absolute paths, they will be kept as is or
578 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
579 the config files will be copied there as temporary files before applying.
581 log.info("Setting arnied configuration")
582 wait_for_arnied(timeout=timeout, vm=vm)
584 config_paths = prep_config_paths(config_files)
585 for config_path in config_paths:
586 with open(config_path, "rt", errors='replace') as config:
587 log.debug("Contents of applied %s:\n%s", config_path, config.read())
589 new_config_path = generate_config_path()
590 vm.copy_files_to(config_path, new_config_path)
591 config_path = new_config_path
592 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
594 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
595 logging.debug(result)
596 if result.returncode != 0:
597 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
599 " on %s" % vm.name if vm is not None else "",
603 wait_for_generate(vm=vm)
604 except Exception as ex:
605 # handle cases of remote configuration that leads to connection meltdown
606 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
607 log.info("Resetting connection to %s", vm.name)
608 vm.session = vm.wait_for_login(timeout=10)
609 log.debug("Connection reset via remote error: %s", ex)
614 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
615 kind="cnf", timeout=30, vm=None):
617 Perform semi-dynamic arnied configuration from an updated version of the
620 :param config_files: config files to use for the configuration
621 :type config_files: [str]
622 :param params_dict: parameters to override the defaults in the config files
623 :type params_dict: {str, str}
624 :param regex_dict: regular expressions to use for matching the overriden parameters
625 :type regex_dict: {str, str} or None
626 :param str kind: "json" or "cnf"
627 :param int timeout: arnied run verification timeout
628 :param vm: vm to run on if running on a guest instead of the host
629 :type vm: :py:class:`virttest.qemu_vm.VM` or None
631 The config files must be provided and are always expected to be found on
632 the host. If these are absolute paths, they will be kept as is or
633 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
634 the config files will be copied there as temporary files before applying.
636 log.info("Performing semi-dynamic arnied configuration")
638 config_paths = prep_cnf(config_files, params_dict, regex_dict)
639 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
641 log.info("Semi-dynamic arnied configuration successful!")
644 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
646 Perform dynamic arnied configuration from fully generated config files.
648 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
649 :type cnf: {str, str}
650 :param config_file: optional user supplied filename
651 :type config_file: str or None
652 :param str kind: "json", "cnf", or "raw"
653 :param int timeout: arnied run verification timeout
654 :param vm: vm to run on if running on a guest instead of the host
655 :type vm: :py:class:`virttest.qemu_vm.VM` or None
656 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
657 :raises: :py:class:`ConfigError` if cannot apply file
659 The config file might not be provided in which case a temporary file will
660 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
661 an absolute path. If a vm is provided, the config file will be copied there
662 as a temporary file before applying.
664 if config_file is None:
665 config_path = generate_config_path(dumped=True)
666 elif os.path.isabs(config_file):
667 config_path = config_file
669 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
670 generated = config_file is None
671 config_file = os.path.basename(config_path)
672 log.info("Using %s cnf file %s%s",
673 "generated" if generated else "user-supplied",
674 config_file, " on %s" % vm.name if vm is not None else "")
676 # Important to write bytes here to ensure text is encoded with latin-1
677 fd = open(config_path, "wb")
680 "raw": cnfvar.write_cnf_raw,
681 "json": cnfvar.write_cnf_json,
682 "cnf": cnfvar.write_cnf
684 SET_CNF_METHODS[kind](cnf, out=fd)
686 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
690 log.info("Generated config file %s", config_path)
692 kind = "cnf" if kind != "json" else kind
693 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
696 def set_cnf_pipe(cnf, timeout=30, block=False):
698 Set local configuration by talking to arnied via ``set_cnf``.
700 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
701 :type cnf: {str, str}
702 :param int timeout: arnied run verification timeout
703 :param bool block: whether to wait for generate to complete the
705 :returns: whether ``set_cnf`` succeeded or not
708 This is obviously not generic but supposed to be run on the guest.
710 log.info("Setting arnied configuration through local pipe")
711 wait_for_arnied(timeout=timeout)
713 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
716 log.error("Error applying configuration; status=%r" % exit)
717 log.error("and stderr:\n%s" % out)
719 log.debug("Configuration successfully passed to set_cnf, "
720 "read %d B from pipe" % len(out))
723 log.debug("Waiting for config job to complete")
726 log.debug("Exiting sucessfully")
730 def prep_config_paths(config_files, config_dir=None):
732 Prepare absolute paths for all configs at an expected location.
734 :param config_files: config files to use for the configuration
735 :type config_files: [str]
736 :param config_dir: config directory to prepend to the filepaths
737 :type config_dir: str or None
738 :returns: list of the full config paths
741 if config_dir is None:
742 config_dir = SRC_CONFIG_DIR
744 for config_file in config_files:
745 if os.path.isabs(config_file):
746 # Absolute path: The user requested a specific file
747 # f.e. needed for dynamic arnied config update
748 config_path = config_file
750 config_path = os.path.join(os.path.abspath(config_dir),
752 logging.debug("Using %s for original path %s", config_path, config_file)
753 config_paths.append(config_path)
757 def prep_cnf_value(config_file, value,
758 regex=None, template_key=None, ignore_fail=False):
760 Replace value in a provided arnied config file.
762 :param str config_file: file to use for the replacement
763 :param str value: value to replace the first matched group with
764 :param regex: regular expression to use when replacing a cnf value
765 :type regex: str or None
766 :param template_key: key of a quick template to use for the regex
767 :type template_key: str or None
768 :param bool ignore_fail: whether to ignore regex mismatching
769 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
771 In order to ensure better matching capabilities you are supposed to
772 provide a regex pattern with at least one subgroup to match your value.
773 What this means is that the value you like to replace is not directly
774 searched into the config text but matched within a larger regex in
775 in order to avoid any mismatch.
778 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
780 if template_key is None:
781 pattern = regex.encode()
783 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
784 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
785 pattern = samples[template_key].encode()
787 with open(config_file, "rb") as file_handle:
788 text = file_handle.read()
789 match_line = re.search(pattern, text)
791 if match_line is None and not ignore_fail:
792 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
793 elif match_line is not None:
794 old_line = match_line.group(0)
795 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
796 line = re.search(pattern, text).group(0)
797 log.debug("Updating %s to %s in %s", old_line, line, config_file)
798 with open(config_file, "wb") as file_handle:
799 file_handle.write(text)
802 def prep_cnf(config_files, params_dict, regex_dict=None):
804 Update all config files with the default overriding parameters,
805 i.e. override the values hard-coded in those config files.
807 :param config_files: config files to use for the configuration
808 :type config_files: [str]
809 :param params_dict: parameters to override the defaults in the config files
810 :type params_dict: {str, str}
811 :param regex_dict: regular expressions to use for matching the overriden parameters
812 :type regex_dict: {str, str} or None
813 :returns: list of prepared (modified) config paths
816 log.info("Preparing %s template config files", len(config_files))
818 src_config_paths = prep_config_paths(config_files)
819 new_config_paths = []
820 for config_path in src_config_paths:
821 new_config_path = generate_config_path(dumped=True)
822 shutil.copy(config_path, new_config_path)
823 new_config_paths.append(new_config_path)
825 for config_path in new_config_paths:
826 for param_key in params_dict.keys():
827 if regex_dict is None:
828 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
829 elif param_key in regex_dict.keys():
830 regex_val = regex_dict[param_key] % param_key.upper()
831 elif re.match("\w*_\d+$", param_key):
832 final_parameter, parent_id = \
833 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
834 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
835 % (parent_id, final_parameter.upper())
836 log.debug("Requested regex for %s is '%s'",
837 param_key, regex_val)
839 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
840 prep_cnf_value(config_path, params_dict[param_key],
841 regex=regex_val, ignore_fail=True)
842 log.info("Prepared template config file %s", config_path)
844 return new_config_paths
847 def generate_config_path(dumped=False):
849 Generate path for a temporary config name.
851 :param bool dumped: whether the file should be in the dump
852 directory or in temporary directory
853 :returns: generated config file path
856 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
857 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
870 def batch_update_cnf(cnf, vars):
872 Perform a batch update of multiple cnf variables.
874 :param cnf: CNF variable to update
875 :type cnf: BuildCnfVar object
876 :param vars: tuples of enumerated action and subtuple with data
877 :type vars: [(int, (str, int, str))]
878 :returns: updated CNF variable
879 :rtype: BuildCnfVar object
881 The actions are indexed in the same order: delete, update, add, child.
884 for (action, data) in vars:
887 last = cnf.update_cnf(var, ref, val)
890 last = cnf.add_cnf(var, ref, val)
891 elif action == Delete:
892 last = cnf.del_cnf(data)
893 elif action == Child: # only one depth supported
896 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
900 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
902 Build a CNF variable and save it in a config file.
904 :param str kind: name of the CNF variable
905 :param int instance: instance number of the CNF variable
906 :param vals: tuples of enumerated action and subtuple with data
907 :type vals: [(int, (str, int, str))]
908 :param str data: data for the CNF variable
909 :param filename: optional custom name of the config file
910 :type filename: str or None
911 :returns: name of the saved config file
914 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
915 batch_update_cnf(builder, vals)
916 filename = generate_config_path(dumped=True) if filename is None else filename
917 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
918 builder.save(filename)