1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 Copyright: Intra2net AG
30 There are three types of setting some cnfvar configuration:
32 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
33 config file without modifying its content in any way
34 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
35 static file but rather as a template, replacing regex-matched values to
36 adapt it to different configurations
37 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
38 and custom cnfvar classes and writing them into config files of a desired
39 format (json, cnf, or raw)
43 ------------------------------------------------------
55 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
57 from .cnfline import build_cnfvar
63 #: default set_cnf binary
64 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
65 #: default arnied_helper binary
66 BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
67 #: default location for template configuration files
69 #: default location for dumped configuration files
73 class ConfigError(Exception):
77 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
79 Universal command run wrapper.
81 :param str cmd: command to run
82 :param bool ignore_errors: whether not to raise error on command failure
83 :param vm: vm to run on if running on a guest instead of the host
84 :type vm: :py:class:`virttest.qemu_vm.VM` or None
85 :param int timeout: amount of seconds to wait for the program to run
86 :returns: command result output
88 :raises: :py:class:`OSError` if command failed and cannot be ignored
91 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
92 stdout = stdout.encode()
98 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
99 return subprocess.CompletedProcess(cmd, status, stdout=stdout, stderr=stderr)
101 return subprocess.run(cmd, check=not ignore_errors, shell=True, capture_output=True)
104 def verify_running(process='arnied', timeout=60, vm=None):
106 Verify if a given process is running via 'pgrep'.
108 :param str process: process to verify if running
109 :param int timeout: run verification timeout
110 :param vm: vm to run on if running on a guest instead of the host
111 :type vm: :py:class:`virttest.qemu_vm.VM` or None
112 :raises: :py:class:`RuntimeError` if process is not running
117 platform_str = " on %s" % vm.name
118 for i in range(timeout):
119 log.info("Checking whether %s is running%s (%i\%i)",
120 process, platform_str, i, timeout)
121 result = run_cmd(cmd="pgrep -l -x %s" % process,
122 ignore_errors=True, vm=vm)
123 if result.returncode == 0:
127 raise RuntimeError("Process %s does not seem to be running" % process)
130 # Basic functionality
133 def accept_licence(vm=None):
135 Accept the Intra2net license.
137 :param vm: vm to run on if running on a guest instead of the host
138 :type vm: :py:class:`virttest.qemu_vm.VM` or None
140 This is mostly useful for simplified webpage access.
142 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
143 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
145 wait_for_generate(vm=vm)
148 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
150 Go online with the given provider id.
152 :param provider_id: provider to go online with
153 :type provider_id: int
154 :param wait_online: whether to wait until online
155 :type wait_online: bool
156 :param vm: vm to run on if running on a guest instead of the host
157 :type vm: :py:class:`virttest.qemu_vm.VM` or None
159 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
161 log.info("Switching to online mode with provider %d", provider_id)
163 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
164 if b'PROVIDER,' not in get_cnf_res.stdout:
165 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
169 cmd = 'tell-connd --online P%i' % provider_id
170 result = run_cmd(cmd=cmd, vm=vm)
174 wait_for_online(provider_id, timeout=timeout, vm=vm)
177 def go_offline(wait_offline=True, vm=None):
181 :param wait_offline: whether to wait until offline
182 :type wait_offline: bool
183 :param vm: vm to run on if running on a guest instead of the host
184 :type vm: :py:class:`virttest.qemu_vm.VM` or None
186 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
188 cmd = 'tell-connd --offline'
189 result = run_cmd(cmd=cmd, vm=vm)
193 if wait_offline is True:
194 wait_for_offline(vm=vm)
196 wait_for_offline(wait_offline, vm=vm)
199 def wait_for_offline(timeout=60, vm=None):
201 Wait for arnied to signal we are offline.
203 :param int timeout: maximum timeout for waiting
204 :param vm: vm to run on if running on a guest instead of the host
205 :type vm: :py:class:`virttest.qemu_vm.VM` or None
207 _wait_for_online_status('offline', None, timeout, vm)
210 def wait_for_online(provider_id, timeout=60, vm=None):
212 Wait for arnied to signal we are online.
214 :param provider_id: provider to go online with
215 :type provider_id: int
216 :param int timeout: maximum timeout for waiting
217 :param vm: vm to run on if running on a guest instead of the host
218 :type vm: :py:class:`virttest.qemu_vm.VM` or None
220 _wait_for_online_status('online', provider_id, timeout, vm)
223 def _wait_for_online_status(status, provider_id, timeout, vm):
224 # Don't use tell-connd --status here since the actual
225 # ONLINE signal to arnied is transmitted
226 # asynchronously via arnieclient_muxer.
228 if status == 'online':
229 expected_output = 'DEFAULT: 2'
230 set_status_func = lambda: go_online(provider_id, False, vm)
231 elif status == 'offline':
232 expected_output = 'DEFAULT: 0'
233 set_status_func = lambda: go_offline(False, vm)
235 raise ValueError('expect status "online" or "offline", not "{0}"!'
238 log.info("Waiting for arnied to be {0} within {1} seconds"
239 .format(status, timeout))
241 for i in range(timeout):
242 # arnied might invalidate the connd "connection barrier"
243 # after generate was running and switch to OFFLINE (race condition).
244 # -> tell arnied every ten seconds to go online again
245 if i % 10 == 0 and i != 0:
248 cmd = '/usr/intranator/bin/get_var ONLINE'
249 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
252 if expected_output in result.stdout.decode():
253 log.info("arnied is {0}. Continuing.".format(status))
258 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
259 .format(status, timeout))
262 def disable_virscan(vm=None):
264 Disable virscan that could block GENERATE and thus all configurations.
266 :param vm: vm to run on if running on a guest instead of the host
267 :type vm: :py:class:`virttest.qemu_vm.VM` or None
269 log.info("Disabling virus database update")
270 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
272 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
273 result = run_cmd(cmd=cmd, vm=vm)
276 # TODO: this intervention should be solved in later arnied_helper tool
277 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
278 result = run_cmd(cmd=cmd, vm=vm)
280 log.info("Virus database update disabled")
283 def email_transfer(vm=None):
285 Transfer all the emails using the guest tool arnied_helper.
287 :param vm: vm to run on if running on a guest instead of the host
288 :type vm: :py:class:`virttest.qemu_vm.VM` or None
290 cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
291 result = run_cmd(cmd=cmd, vm=vm)
295 def wait_for_email_transfer(timeout=300, vm=None):
297 Wait until the mail queue is empty and all emails are sent.
299 :param int timeout: email transfer timeout
300 :param vm: vm to run on if running on a guest instead of the host
301 :type vm: :py:class:`virttest.qemu_vm.VM` or None
303 for i in range(timeout):
305 # Retrigger mail queue in case something is deferred
306 # by an amavisd-new reconfiguration
307 run_cmd(cmd='postqueue -f', vm=vm)
308 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
310 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
311 log.debug('SMTP queue is empty')
314 log.warning('Timeout reached but SMTP queue still not empty after {} s'
318 def schedule(program, exec_time=0, optional_args="", vm=None):
320 Schedule a program to be executed at a given unix time stamp.
322 :param str program: program whose execution is scheduled
323 :param int exec_time: scheduled time of program's execution
324 :param str optional_args: optional command line arguments
325 :param vm: vm to run on if running on a guest instead of the host
326 :type vm: :py:class:`virttest.qemu_vm.VM` or None
328 log.info("Scheduling %s to be executed at %i", program, exec_time)
329 schedule_dir = "/var/intranator/schedule"
330 # clean previous schedules of the same program
331 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
332 for file_name in files:
333 if file_name.startswith(program.upper()):
334 log.debug("Removing previous scheduled %s", file_name)
336 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
338 os.unlink(os.path.join(schedule_dir, file_name))
340 contents = "%i\n%s\n" % (exec_time, optional_args)
342 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
343 prefix=program.upper() + "_",
346 log.debug("Created temporary file %s", tmp_file.name)
347 tmp_file.write(contents)
349 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
352 vm.copy_files_to(tmp_file.name, moved_tmp_file)
353 os.remove(tmp_file.name)
355 shutil.move(tmp_file.name, moved_tmp_file)
357 log.debug("Moved temporary file to %s", moved_tmp_file)
360 def wait_for_run(program, timeout=300, retries=10, vm=None):
362 Wait for a program using the guest arnied_helper tool.
364 :param str program: scheduled or running program to wait for
365 :param int timeout: program run timeout
366 :param int retries: number of tries to verify that the program is scheduled or running
367 :param vm: vm to run on if running on a guest instead of the host
368 :type vm: :py:class:`virttest.qemu_vm.VM` or None
370 log.info("Waiting for program %s to finish with timeout %i",
372 for i in range(retries):
373 cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
375 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
376 if check_scheduled.returncode == 0:
380 log.warning("The program %s was not scheduled and is not running", program)
382 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
383 f"{program.upper()} --wait-for-program-timeout {timeout}"
384 # add one second to make sure arnied_helper is finished when we expire
385 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
386 log.debug(result.stdout)
389 def wait_for_arnied(timeout=60, vm=None):
391 Wait for arnied socket to be ready.
393 :param int timeout: maximum number of seconds to wait
394 :param vm: vm to run on if running on a guest instead of the host
395 :type vm: :py:class:`virttest.qemu_vm.VM` or None
397 cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
398 f"--wait-for-arnied-socket-timeout {timeout}"
399 # add one second to make sure arnied_helper is finished when we expire
400 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
401 log.debug(result.stdout)
404 # Configuration functionality
406 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
408 Query arnied for a `cnf_key` and extract some information via regex.
410 :param str cnf_key: queried cnf key
411 :param int cnf_index: index of the cnf key
412 :param str regex: regex to apply on the queried cnf key data
413 :param bool compact: whether to retrieve compact version of the matched cnf keys
414 :param int timeout: arnied run verification timeout
415 :param vm: vm to run on if running on a guest instead of the host
416 :type vm: :py:class:`virttest.qemu_vm.VM` or None
417 :returns: extracted information via the regex
420 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
422 wait_for_arnied(timeout=timeout, vm=vm)
425 platform_str = " from %s" % vm.name
426 log.info("Extracting arnied value %s for %s%s using pattern %s",
427 cnf_index, cnf_key, platform_str, regex)
428 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
429 " %s" % cnf_index if cnf_index != -1 else "")
430 output = run_cmd(cmd=cmd, vm=vm).stdout.decode()
431 return re.search(regex, output, flags=re.DOTALL)
434 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
436 Get the id of a configuration of type `cnf_key` and name `value`.
438 :param str cnf_key: queried cnf key
439 :param str value: cnf value of the cnf key
440 :param int timeout: arnied run verification timeout
441 :param vm: vm to run on if running on a guest instead of the host
442 :type vm: :py:class:`virttest.qemu_vm.VM` or None
443 :returns: the cnf id or -1 if no such cnf variable
446 wait_for_arnied(timeout=timeout, vm=vm)
447 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
448 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
452 cnf_id = int(cnf_id.group(1))
453 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
457 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
459 Invoke get_cnf and return a nested CNF structure.
461 :param str varname: "varname" field of the CNF_VAR to look up
462 :param instance: "instance" of that variable to return
464 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
465 :param int timeout: arnied run verification timeout
466 :param vm: vm to run on if running on a guest instead of the host
467 :type vm: :py:class:`virttest.qemu_vm.VM` or None
468 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
469 :rtype: cnfvar option
471 wait_for_arnied(timeout=timeout, vm=vm)
472 # firstly, build argv for get_cnf
473 cmd = ["get_cnf", "-j"]
474 if varname is not None:
475 cmd.append("%s" % varname)
477 cmd.append("%d" % instance)
478 cmd_line = " ".join(cmd)
481 result = run_cmd(cmd=cmd_line, vm=vm)
482 (status, raw) = result.returncode, result.stdout
484 log.info("error %d executing \"%s\"", status, cmd_line)
488 # reading was successful, attempt to parse what we got
490 # The output from "get_cnf -j" is already utf-8. This contrast with
491 # the output of "get_cnf" (no json) which is latin1.
492 if isinstance(raw, bytes):
493 raw = raw.decode("utf-8")
494 cnf = cnfvar.read_cnf_json(raw)
495 except TypeError as exn:
496 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
498 except cnfvar.InvalidCNF as exn:
499 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
503 return cnfvar.get_vars(cnf, data=data)
508 def get_cnfvar_id(varname, data, timeout=30, vm=None):
510 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
512 :param str varname: "varname" field of the CNF_VAR to look up
513 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
514 :param int timeout: arnied run verification timeout
515 :param vm: vm to run on if running on a guest instead of the host
516 :type vm: :py:class:`virttest.qemu_vm.VM` or None
517 :returns: the cnf id or -1 if no such cnf variable
520 wait_for_arnied(timeout=timeout, vm=vm)
521 log.info("Extracting from arnied CNF_VAR %s with data %s",
523 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
524 variables = cnf["cnf"]
525 if len(variables) == 0:
526 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
529 first_instance = int(variables[0]["instance"])
530 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
531 len(variables), first_instance)
532 return first_instance
535 def wait_for_generate(timeout=300, vm=None):
537 Wait for the 'generate' program to complete.
539 Arguments are similar to the ones from :py:method:`wait_for_run`.
541 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
542 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
545 def unset_cnf(varname="", instance="", timeout=30, vm=None):
547 Remove configuration from arnied.
549 :param str varname: "varname" field of the CNF_VAR to unset
550 :param int instance: "instance" of that variable to unset
551 :param int timeout: arnied run verification timeout
552 :param vm: vm to run on if running on a guest instead of the host
553 :type vm: :py:class:`virttest.qemu_vm.VM` or None
555 wait_for_arnied(timeout=timeout, vm=vm)
557 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
558 run_cmd(cmd=cmd, vm=vm)
560 wait_for_generate(vm=vm)
563 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
565 Perform static arnied configuration through a set of config files.
567 :param config_files: config files to use for the configuration
568 :type config_files: [str]
569 :param str kind: "json" or "cnf"
570 :param int timeout: arnied run verification timeout
571 :param vm: vm to run on if running on a guest instead of the host
572 :type vm: :py:class:`virttest.qemu_vm.VM` or None
573 :raises: :py:class:`ConfigError` if cannot apply file
575 The config files must be provided and are always expected to be found on
576 the host. If these are absolute paths, they will be kept as is or
577 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
578 the config files will be copied there as temporary files before applying.
580 log.info("Setting arnied configuration")
581 wait_for_arnied(timeout=timeout, vm=vm)
583 config_paths = prep_config_paths(config_files)
584 for config_path in config_paths:
585 with open(config_path, "rt", errors='replace') as config:
586 log.debug("Contents of applied %s:\n%s", config_path, config.read())
588 new_config_path = generate_config_path()
589 vm.copy_files_to(config_path, new_config_path)
590 config_path = new_config_path
591 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
593 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
594 logging.debug(result)
595 if result.returncode != 0:
596 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
598 " on %s" % vm.name if vm is not None else "",
602 wait_for_generate(vm=vm)
603 except Exception as ex:
604 # handle cases of remote configuration that leads to connection meltdown
605 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
606 log.info("Resetting connection to %s", vm.name)
607 vm.session = vm.wait_for_login(timeout=10)
608 log.debug("Connection reset via remote error: %s", ex)
613 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
614 kind="cnf", timeout=30, vm=None):
616 Perform semi-dynamic arnied configuration from an updated version of the
619 :param config_files: config files to use for the configuration
620 :type config_files: [str]
621 :param params_dict: parameters to override the defaults in the config files
622 :type params_dict: {str, str}
623 :param regex_dict: regular expressions to use for matching the overriden parameters
624 :type regex_dict: {str, str} or None
625 :param str kind: "json" or "cnf"
626 :param int timeout: arnied run verification timeout
627 :param vm: vm to run on if running on a guest instead of the host
628 :type vm: :py:class:`virttest.qemu_vm.VM` or None
630 The config files must be provided and are always expected to be found on
631 the host. If these are absolute paths, they will be kept as is or
632 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
633 the config files will be copied there as temporary files before applying.
635 log.info("Performing semi-dynamic arnied configuration")
637 config_paths = prep_cnf(config_files, params_dict, regex_dict)
638 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
640 log.info("Semi-dynamic arnied configuration successful!")
643 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
645 Perform dynamic arnied configuration from fully generated config files.
647 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
648 :type cnf: {str, str}
649 :param config_file: optional user supplied filename
650 :type config_file: str or None
651 :param str kind: "json", "cnf", or "raw"
652 :param int timeout: arnied run verification timeout
653 :param vm: vm to run on if running on a guest instead of the host
654 :type vm: :py:class:`virttest.qemu_vm.VM` or None
655 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
656 :raises: :py:class:`ConfigError` if cannot apply file
658 The config file might not be provided in which case a temporary file will
659 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
660 an absolute path. If a vm is provided, the config file will be copied there
661 as a temporary file before applying.
663 if config_file is None:
664 config_path = generate_config_path(dumped=True)
665 elif os.path.isabs(config_file):
666 config_path = config_file
668 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
669 generated = config_file is None
670 config_file = os.path.basename(config_path)
671 log.info("Using %s cnf file %s%s",
672 "generated" if generated else "user-supplied",
673 config_file, " on %s" % vm.name if vm is not None else "")
675 # Important to write bytes here to ensure text is encoded with latin-1
676 fd = open(config_path, "wb")
679 "raw": cnfvar.write_cnf_raw,
680 "json": cnfvar.write_cnf_json,
681 "cnf": cnfvar.write_cnf
683 SET_CNF_METHODS[kind](cnf, out=fd)
685 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
689 log.info("Generated config file %s", config_path)
691 kind = "cnf" if kind != "json" else kind
692 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
695 def set_cnf_pipe(cnf, timeout=30, block=False):
697 Set local configuration by talking to arnied via ``set_cnf``.
699 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
700 :type cnf: {str, str}
701 :param int timeout: arnied run verification timeout
702 :param bool block: whether to wait for generate to complete the
704 :returns: whether ``set_cnf`` succeeded or not
707 This is obviously not generic but supposed to be run on the guest.
709 log.info("Setting arnied configuration through local pipe")
710 wait_for_arnied(timeout=timeout)
712 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
715 log.error("Error applying configuration; status=%r" % exit)
716 log.error("and stderr:\n%s" % out)
718 log.debug("Configuration successfully passed to set_cnf, "
719 "read %d B from pipe" % len(out))
722 log.debug("Waiting for config job to complete")
725 log.debug("Exiting sucessfully")
729 def prep_config_paths(config_files, config_dir=None):
731 Prepare absolute paths for all configs at an expected location.
733 :param config_files: config files to use for the configuration
734 :type config_files: [str]
735 :param config_dir: config directory to prepend to the filepaths
736 :type config_dir: str or None
737 :returns: list of the full config paths
740 if config_dir is None:
741 config_dir = SRC_CONFIG_DIR
743 for config_file in config_files:
744 if os.path.isabs(config_file):
745 # Absolute path: The user requested a specific file
746 # f.e. needed for dynamic arnied config update
747 config_path = config_file
749 config_path = os.path.join(os.path.abspath(config_dir),
751 logging.debug("Using %s for original path %s", config_path, config_file)
752 config_paths.append(config_path)
756 def prep_cnf_value(config_file, value,
757 regex=None, template_key=None, ignore_fail=False):
759 Replace value in a provided arnied config file.
761 :param str config_file: file to use for the replacement
762 :param str value: value to replace the first matched group with
763 :param regex: regular expression to use when replacing a cnf value
764 :type regex: str or None
765 :param template_key: key of a quick template to use for the regex
766 :type template_key: str or None
767 :param bool ignore_fail: whether to ignore regex mismatching
768 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
770 In order to ensure better matching capabilities you are supposed to
771 provide a regex pattern with at least one subgroup to match your value.
772 What this means is that the value you like to replace is not directly
773 searched into the config text but matched within a larger regex in
774 in order to avoid any mismatch.
777 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
779 if template_key is None:
780 pattern = regex.encode()
782 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
783 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
784 pattern = samples[template_key].encode()
786 with open(config_file, "rb") as file_handle:
787 text = file_handle.read()
788 match_line = re.search(pattern, text)
790 if match_line is None and not ignore_fail:
791 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
792 elif match_line is not None:
793 old_line = match_line.group(0)
794 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
795 line = re.search(pattern, text).group(0)
796 log.debug("Updating %s to %s in %s", old_line, line, config_file)
797 with open(config_file, "wb") as file_handle:
798 file_handle.write(text)
801 def prep_cnf(config_files, params_dict, regex_dict=None):
803 Update all config files with the default overriding parameters,
804 i.e. override the values hard-coded in those config files.
806 :param config_files: config files to use for the configuration
807 :type config_files: [str]
808 :param params_dict: parameters to override the defaults in the config files
809 :type params_dict: {str, str}
810 :param regex_dict: regular expressions to use for matching the overriden parameters
811 :type regex_dict: {str, str} or None
812 :returns: list of prepared (modified) config paths
815 log.info("Preparing %s template config files", len(config_files))
817 src_config_paths = prep_config_paths(config_files)
818 new_config_paths = []
819 for config_path in src_config_paths:
820 new_config_path = generate_config_path(dumped=True)
821 shutil.copy(config_path, new_config_path)
822 new_config_paths.append(new_config_path)
824 for config_path in new_config_paths:
825 for param_key in params_dict.keys():
826 if regex_dict is None:
827 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
828 elif param_key in regex_dict.keys():
829 regex_val = regex_dict[param_key] % param_key.upper()
830 elif re.match("\w*_\d+$", param_key):
831 final_parameter, parent_id = \
832 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
833 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
834 % (parent_id, final_parameter.upper())
835 log.debug("Requested regex for %s is '%s'",
836 param_key, regex_val)
838 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
839 prep_cnf_value(config_path, params_dict[param_key],
840 regex=regex_val, ignore_fail=True)
841 log.info("Prepared template config file %s", config_path)
843 return new_config_paths
846 def generate_config_path(dumped=False):
848 Generate path for a temporary config name.
850 :param bool dumped: whether the file should be in the dump
851 directory or in temporary directory
852 :returns: generated config file path
855 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
856 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
869 def batch_update_cnf(cnf, vars):
871 Perform a batch update of multiple cnf variables.
873 :param cnf: CNF variable to update
874 :type cnf: BuildCnfVar object
875 :param vars: tuples of enumerated action and subtuple with data
876 :type vars: [(int, (str, int, str))]
877 :returns: updated CNF variable
878 :rtype: BuildCnfVar object
880 The actions are indexed in the same order: delete, update, add, child.
883 for (action, data) in vars:
886 last = cnf.update_cnf(var, ref, val)
889 last = cnf.add_cnf(var, ref, val)
890 elif action == Delete:
891 last = cnf.del_cnf(data)
892 elif action == Child: # only one depth supported
895 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
899 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
901 Build a CNF variable and save it in a config file.
903 :param str kind: name of the CNF variable
904 :param int instance: instance number of the CNF variable
905 :param vals: tuples of enumerated action and subtuple with data
906 :type vals: [(int, (str, int, str))]
907 :param str data: data for the CNF variable
908 :param filename: optional custom name of the config file
909 :type filename: str or None
910 :returns: name of the saved config file
913 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
914 batch_update_cnf(builder, vals)
915 filename = generate_config_path(dumped=True) if filename is None else filename
916 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
917 builder.save(filename)