1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 Copyright: Intra2net AG
30 There are three types of setting some cnfvar configuration:
32 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
33 config file without modifying its content in any way
34 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
35 static file but rather as a template, replacing regex-matched values to
36 adapt it to different configurations
37 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
38 and custom cnfvar classes and writing them into config files of a desired
39 format (json, cnf, or raw)
43 ------------------------------------------------------
55 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
57 from .cnfline import build_cnfvar
63 #: default set_cnf binary
64 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
65 #: default arnied_helper binary
66 BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
67 #: default location for template configuration files
69 #: default location for dumped configuration files
73 class ConfigError(Exception):
77 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
79 Universal command run wrapper.
81 :param str cmd: command to run
82 :param bool ignore_errors: whether not to raise error on command failure
83 :param vm: vm to run on if running on a guest instead of the host
84 :type vm: :py:class:`virttest.qemu_vm.VM` or None
85 :param int timeout: amount of seconds to wait for the program to run
86 :returns: command result output where output (stdout/stderr) is bytes
87 (encoding dependent on environment and command given)
88 :rtype: :py:class:`subprocess.CompletedProcess`
89 :raises: :py:class:`OSError` if command failed and cannot be ignored
92 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
93 stdout = stdout.encode()
99 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
100 return subprocess.CompletedProcess(cmd, status,
101 stdout=stdout, stderr=stderr)
103 return subprocess.run(cmd, check=not ignore_errors, shell=True,
107 def verify_running(process='arnied', timeout=60, vm=None):
109 Verify if a given process is running via 'pgrep'.
111 :param str process: process to verify if running
112 :param int timeout: run verification timeout
113 :param vm: vm to run on if running on a guest instead of the host
114 :type vm: :py:class:`virttest.qemu_vm.VM` or None
115 :raises: :py:class:`RuntimeError` if process is not running
120 platform_str = " on %s" % vm.name
121 for i in range(timeout):
122 log.info("Checking whether %s is running%s (%i\%i)",
123 process, platform_str, i, timeout)
124 result = run_cmd(cmd="pgrep -l -x %s" % process,
125 ignore_errors=True, vm=vm)
126 if result.returncode == 0:
130 raise RuntimeError("Process %s does not seem to be running" % process)
133 # Basic functionality
136 def accept_licence(vm=None):
138 Accept the Intra2net license.
140 :param vm: vm to run on if running on a guest instead of the host
141 :type vm: :py:class:`virttest.qemu_vm.VM` or None
143 This is mostly useful for simplified webpage access.
145 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
146 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
148 wait_for_generate(vm=vm)
151 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
153 Go online with the given provider id.
155 :param provider_id: provider to go online with
156 :type provider_id: int
157 :param wait_online: whether to wait until online
158 :type wait_online: bool
159 :param vm: vm to run on if running on a guest instead of the host
160 :type vm: :py:class:`virttest.qemu_vm.VM` or None
162 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
164 log.info("Switching to online mode with provider %d", provider_id)
166 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
167 if b'PROVIDER,' not in get_cnf_res.stdout:
168 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
172 cmd = 'tell-connd --online P%i' % provider_id
173 result = run_cmd(cmd=cmd, vm=vm)
177 wait_for_online(provider_id, timeout=timeout, vm=vm)
180 def go_offline(wait_offline=True, vm=None):
184 :param wait_offline: whether to wait until offline
185 :type wait_offline: bool
186 :param vm: vm to run on if running on a guest instead of the host
187 :type vm: :py:class:`virttest.qemu_vm.VM` or None
189 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
191 cmd = 'tell-connd --offline'
192 result = run_cmd(cmd=cmd, vm=vm)
196 if wait_offline is True:
197 wait_for_offline(vm=vm)
199 wait_for_offline(wait_offline, vm=vm)
202 def wait_for_offline(timeout=60, vm=None):
204 Wait for arnied to signal we are offline.
206 :param int timeout: maximum timeout for waiting
207 :param vm: vm to run on if running on a guest instead of the host
208 :type vm: :py:class:`virttest.qemu_vm.VM` or None
210 _wait_for_online_status('offline', None, timeout, vm)
213 def wait_for_online(provider_id, timeout=60, vm=None):
215 Wait for arnied to signal we are online.
217 :param provider_id: provider to go online with
218 :type provider_id: int
219 :param int timeout: maximum timeout for waiting
220 :param vm: vm to run on if running on a guest instead of the host
221 :type vm: :py:class:`virttest.qemu_vm.VM` or None
223 _wait_for_online_status('online', provider_id, timeout, vm)
226 def _wait_for_online_status(status, provider_id, timeout, vm):
227 # Don't use tell-connd --status here since the actual
228 # ONLINE signal to arnied is transmitted
229 # asynchronously via arnieclient_muxer.
231 if status == 'online':
232 expected_output = 'DEFAULT: 2'
233 set_status_func = lambda: go_online(provider_id, False, vm)
234 elif status == 'offline':
235 expected_output = 'DEFAULT: 0'
236 set_status_func = lambda: go_offline(False, vm)
238 raise ValueError('expect status "online" or "offline", not "{0}"!'
241 log.info("Waiting for arnied to be {0} within {1} seconds"
242 .format(status, timeout))
244 for i in range(timeout):
245 # arnied might invalidate the connd "connection barrier"
246 # after generate was running and switch to OFFLINE (race condition).
247 # -> tell arnied every ten seconds to go online again
248 if i % 10 == 0 and i != 0:
251 cmd = '/usr/intranator/bin/get_var ONLINE'
252 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
255 if expected_output in result.stdout.decode():
256 log.info("arnied is {0}. Continuing.".format(status))
261 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
262 .format(status, timeout))
265 def disable_virscan(vm=None):
267 Disable virscan that could block GENERATE and thus all configurations.
269 :param vm: vm to run on if running on a guest instead of the host
270 :type vm: :py:class:`virttest.qemu_vm.VM` or None
272 log.info("Disabling virus database update")
273 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
275 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
276 result = run_cmd(cmd=cmd, vm=vm)
279 # TODO: this intervention should be solved in later arnied_helper tool
280 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
281 result = run_cmd(cmd=cmd, vm=vm)
283 log.info("Virus database update disabled")
286 def email_transfer(vm=None):
288 Transfer all the emails using the guest tool arnied_helper.
290 :param vm: vm to run on if running on a guest instead of the host
291 :type vm: :py:class:`virttest.qemu_vm.VM` or None
293 cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
294 result = run_cmd(cmd=cmd, vm=vm)
298 def wait_for_email_transfer(timeout=300, vm=None):
300 Wait until the mail queue is empty and all emails are sent.
302 :param int timeout: email transfer timeout
303 :param vm: vm to run on if running on a guest instead of the host
304 :type vm: :py:class:`virttest.qemu_vm.VM` or None
306 for i in range(timeout):
308 # Retrigger mail queue in case something is deferred
309 # by an amavisd-new reconfiguration
310 run_cmd(cmd='postqueue -f', vm=vm)
311 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
313 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
314 log.debug('SMTP queue is empty')
317 log.warning('Timeout reached but SMTP queue still not empty after {} s'
321 def schedule(program, exec_time=0, optional_args="", vm=None):
323 Schedule a program to be executed at a given unix time stamp.
325 :param str program: program whose execution is scheduled
326 :param int exec_time: scheduled time of program's execution
327 :param str optional_args: optional command line arguments
328 :param vm: vm to run on if running on a guest instead of the host
329 :type vm: :py:class:`virttest.qemu_vm.VM` or None
331 log.info("Scheduling %s to be executed at %i", program, exec_time)
332 schedule_dir = "/var/intranator/schedule"
333 # clean previous schedules of the same program
334 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
335 for file_name in files:
336 if file_name.startswith(program.upper()):
337 log.debug("Removing previous scheduled %s", file_name)
339 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
341 os.unlink(os.path.join(schedule_dir, file_name))
343 contents = "%i\n%s\n" % (exec_time, optional_args)
345 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
346 prefix=program.upper() + "_",
349 log.debug("Created temporary file %s", tmp_file.name)
350 tmp_file.write(contents)
352 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
355 vm.copy_files_to(tmp_file.name, moved_tmp_file)
356 os.remove(tmp_file.name)
358 shutil.move(tmp_file.name, moved_tmp_file)
360 log.debug("Moved temporary file to %s", moved_tmp_file)
363 def wait_for_run(program, timeout=300, retries=10, vm=None):
365 Wait for a program using the guest arnied_helper tool.
367 :param str program: scheduled or running program to wait for
368 :param int timeout: program run timeout
369 :param int retries: number of tries to verify that the program is scheduled or running
370 :param vm: vm to run on if running on a guest instead of the host
371 :type vm: :py:class:`virttest.qemu_vm.VM` or None
373 log.info("Waiting for program %s to finish with timeout %i",
375 for i in range(retries):
376 cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
378 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
379 if check_scheduled.returncode == 0:
383 log.warning("The program %s was not scheduled and is not running", program)
385 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
386 f"{program.upper()} --wait-for-program-timeout {timeout}"
387 # add one second to make sure arnied_helper is finished when we expire
388 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
389 log.debug(result.stdout)
392 def wait_for_arnied(timeout=60, vm=None):
394 Wait for arnied socket to be ready.
396 :param int timeout: maximum number of seconds to wait
397 :param vm: vm to run on if running on a guest instead of the host
398 :type vm: :py:class:`virttest.qemu_vm.VM` or None
400 cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
401 f"--wait-for-arnied-socket-timeout {timeout}"
402 # add one second to make sure arnied_helper is finished when we expire
403 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
404 log.debug(result.stdout)
407 # Configuration functionality
409 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
411 Query arnied for a `cnf_key` and extract some information via regex.
413 :param str cnf_key: queried cnf key
414 :param int cnf_index: index of the cnf key
415 :param str regex: regex to apply on the queried cnf key data
416 :param bool compact: whether to retrieve compact version of the matched cnf keys
417 :param int timeout: arnied run verification timeout
418 :param vm: vm to run on if running on a guest instead of the host
419 :type vm: :py:class:`virttest.qemu_vm.VM` or None
420 :returns: extracted information via the regex
423 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
425 wait_for_arnied(timeout=timeout, vm=vm)
428 platform_str = " from %s" % vm.name
429 log.info("Extracting arnied value %s for %s%s using pattern %s",
430 cnf_index, cnf_key, platform_str, regex)
431 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
432 " %s" % cnf_index if cnf_index != -1 else "")
433 # get_cnf creates latin1-encoded output, transfer from VM removes non-ascii
434 output = run_cmd(cmd=cmd, vm=vm).stdout.decode('latin1')
435 return re.search(regex, output, flags=re.DOTALL)
438 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
440 Get the id of a configuration of type `cnf_key` and name `value`.
442 :param str cnf_key: queried cnf key
443 :param str value: cnf value of the cnf key
444 :param int timeout: arnied run verification timeout
445 :param vm: vm to run on if running on a guest instead of the host
446 :type vm: :py:class:`virttest.qemu_vm.VM` or None
447 :returns: the cnf id or -1 if no such cnf variable
450 wait_for_arnied(timeout=timeout, vm=vm)
451 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
452 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
456 cnf_id = int(cnf_id.group(1))
457 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
461 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
463 Invoke get_cnf and return a nested CNF structure.
465 :param str varname: "varname" field of the CNF_VAR to look up
466 :param instance: "instance" of that variable to return
468 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
469 :param int timeout: arnied run verification timeout
470 :param vm: vm to run on if running on a guest instead of the host
471 :type vm: :py:class:`virttest.qemu_vm.VM` or None
472 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
473 :rtype: cnfvar option
475 wait_for_arnied(timeout=timeout, vm=vm)
476 # firstly, build argv for get_cnf
477 cmd = ["get_cnf", "-j"]
478 if varname is not None:
479 cmd.append("%s" % varname)
481 cmd.append("%d" % instance)
482 cmd_line = " ".join(cmd)
485 result = run_cmd(cmd=cmd_line, vm=vm)
486 (status, raw) = result.returncode, result.stdout
488 log.info("error %d executing \"%s\"", status, cmd_line)
492 # reading was successful, attempt to parse what we got
494 # The output from "get_cnf -j" is already utf-8. This contrast with
495 # the output of "get_cnf" (no json) which is latin1.
496 if isinstance(raw, bytes):
497 raw = raw.decode("utf-8")
498 cnf = cnfvar.read_cnf_json(raw)
499 except TypeError as exn:
500 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
502 except cnfvar.InvalidCNF as exn:
503 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
507 return cnfvar.get_vars(cnf, data=data)
512 def get_cnfvar_id(varname, data, timeout=30, vm=None):
514 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
516 :param str varname: "varname" field of the CNF_VAR to look up
517 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
518 :param int timeout: arnied run verification timeout
519 :param vm: vm to run on if running on a guest instead of the host
520 :type vm: :py:class:`virttest.qemu_vm.VM` or None
521 :returns: the cnf id or -1 if no such cnf variable
524 wait_for_arnied(timeout=timeout, vm=vm)
525 log.info("Extracting from arnied CNF_VAR %s with data %s",
527 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
528 variables = cnf["cnf"]
529 if len(variables) == 0:
530 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
533 first_instance = int(variables[0]["instance"])
534 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
535 len(variables), first_instance)
536 return first_instance
539 def wait_for_generate(timeout=300, vm=None):
541 Wait for the 'generate' program to complete.
543 Arguments are similar to the ones from :py:method:`wait_for_run`.
545 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
546 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
549 def unset_cnf(varname="", instance="", timeout=30, vm=None):
551 Remove configuration from arnied.
553 :param str varname: "varname" field of the CNF_VAR to unset
554 :param int instance: "instance" of that variable to unset
555 :param int timeout: arnied run verification timeout
556 :param vm: vm to run on if running on a guest instead of the host
557 :type vm: :py:class:`virttest.qemu_vm.VM` or None
559 wait_for_arnied(timeout=timeout, vm=vm)
561 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
562 run_cmd(cmd=cmd, vm=vm)
564 wait_for_generate(vm=vm)
567 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
569 Perform static arnied configuration through a set of config files.
571 :param config_files: config files to use for the configuration
572 :type config_files: [str]
573 :param str kind: "json" or "cnf"
574 :param int timeout: arnied run verification timeout
575 :param vm: vm to run on if running on a guest instead of the host
576 :type vm: :py:class:`virttest.qemu_vm.VM` or None
577 :raises: :py:class:`ConfigError` if cannot apply file
579 The config files must be provided and are always expected to be found on
580 the host. If these are absolute paths, they will be kept as is or
581 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
582 the config files will be copied there as temporary files before applying.
584 log.info("Setting arnied configuration")
585 wait_for_arnied(timeout=timeout, vm=vm)
587 config_paths = prep_config_paths(config_files)
588 for config_path in config_paths:
589 with open(config_path, "rt", errors='replace') as config:
590 log.debug("Contents of applied %s:\n%s", config_path, config.read())
592 new_config_path = generate_config_path()
593 vm.copy_files_to(config_path, new_config_path)
594 config_path = new_config_path
595 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
597 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
598 logging.debug(result)
599 if result.returncode != 0:
600 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
602 " on %s" % vm.name if vm is not None else "",
606 wait_for_generate(vm=vm)
607 except Exception as ex:
608 # handle cases of remote configuration that leads to connection meltdown
609 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
610 log.info("Resetting connection to %s", vm.name)
611 vm.session = vm.wait_for_login(timeout=10)
612 log.debug("Connection reset via remote error: %s", ex)
617 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
618 kind="cnf", timeout=30, vm=None):
620 Perform semi-dynamic arnied configuration from an updated version of the
623 :param config_files: config files to use for the configuration
624 :type config_files: [str]
625 :param params_dict: parameters to override the defaults in the config files
626 :type params_dict: {str, str}
627 :param regex_dict: regular expressions to use for matching the overriden parameters
628 :type regex_dict: {str, str} or None
629 :param str kind: "json" or "cnf"
630 :param int timeout: arnied run verification timeout
631 :param vm: vm to run on if running on a guest instead of the host
632 :type vm: :py:class:`virttest.qemu_vm.VM` or None
634 The config files must be provided and are always expected to be found on
635 the host. If these are absolute paths, they will be kept as is or
636 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
637 the config files will be copied there as temporary files before applying.
639 log.info("Performing semi-dynamic arnied configuration")
641 config_paths = prep_cnf(config_files, params_dict, regex_dict)
642 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
644 log.info("Semi-dynamic arnied configuration successful!")
647 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
649 Perform dynamic arnied configuration from fully generated config files.
651 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
652 :type cnf: {str, str}
653 :param config_file: optional user supplied filename
654 :type config_file: str or None
655 :param str kind: "json", "cnf", or "raw"
656 :param int timeout: arnied run verification timeout
657 :param vm: vm to run on if running on a guest instead of the host
658 :type vm: :py:class:`virttest.qemu_vm.VM` or None
659 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
660 :raises: :py:class:`ConfigError` if cannot apply file
662 The config file might not be provided in which case a temporary file will
663 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
664 an absolute path. If a vm is provided, the config file will be copied there
665 as a temporary file before applying.
667 if config_file is None:
668 config_path = generate_config_path(dumped=True)
669 elif os.path.isabs(config_file):
670 config_path = config_file
672 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
673 generated = config_file is None
674 config_file = os.path.basename(config_path)
675 log.info("Using %s cnf file %s%s",
676 "generated" if generated else "user-supplied",
677 config_file, " on %s" % vm.name if vm is not None else "")
679 # Important to write bytes here to ensure text is encoded with latin-1
680 fd = open(config_path, "wb")
683 "raw": cnfvar.write_cnf_raw,
684 "json": cnfvar.write_cnf_json,
685 "cnf": cnfvar.write_cnf
687 SET_CNF_METHODS[kind](cnf, out=fd)
689 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
693 log.info("Generated config file %s", config_path)
695 kind = "cnf" if kind != "json" else kind
696 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
699 def set_cnf_pipe(cnf, timeout=30, block=False):
701 Set local configuration by talking to arnied via ``set_cnf``.
703 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
704 :type cnf: {str, str}
705 :param int timeout: arnied run verification timeout
706 :param bool block: whether to wait for generate to complete the
708 :returns: whether ``set_cnf`` succeeded or not
711 This is obviously not generic but supposed to be run on the guest.
713 log.info("Setting arnied configuration through local pipe")
714 wait_for_arnied(timeout=timeout)
716 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
719 log.error("Error applying configuration; status=%r" % exit)
720 log.error("and stderr:\n%s" % out)
722 log.debug("Configuration successfully passed to set_cnf, "
723 "read %d B from pipe" % len(out))
726 log.debug("Waiting for config job to complete")
729 log.debug("Exiting sucessfully")
733 def prep_config_paths(config_files, config_dir=None):
735 Prepare absolute paths for all configs at an expected location.
737 :param config_files: config files to use for the configuration
738 :type config_files: [str]
739 :param config_dir: config directory to prepend to the filepaths
740 :type config_dir: str or None
741 :returns: list of the full config paths
744 if config_dir is None:
745 config_dir = SRC_CONFIG_DIR
747 for config_file in config_files:
748 if os.path.isabs(config_file):
749 # Absolute path: The user requested a specific file
750 # f.e. needed for dynamic arnied config update
751 config_path = config_file
753 config_path = os.path.join(os.path.abspath(config_dir),
755 logging.debug("Using %s for original path %s", config_path, config_file)
756 config_paths.append(config_path)
760 def prep_cnf_value(config_file, value,
761 regex=None, template_key=None, ignore_fail=False):
763 Replace value in a provided arnied config file.
765 :param str config_file: file to use for the replacement
766 :param str value: value to replace the first matched group with
767 :param regex: regular expression to use when replacing a cnf value
768 :type regex: str or None
769 :param template_key: key of a quick template to use for the regex
770 :type template_key: str or None
771 :param bool ignore_fail: whether to ignore regex mismatching
772 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
774 In order to ensure better matching capabilities you are supposed to
775 provide a regex pattern with at least one subgroup to match your value.
776 What this means is that the value you like to replace is not directly
777 searched into the config text but matched within a larger regex in
778 in order to avoid any mismatch.
781 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
783 if template_key is None:
784 pattern = regex.encode()
786 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
787 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
788 pattern = samples[template_key].encode()
790 with open(config_file, "rb") as file_handle:
791 text = file_handle.read()
792 match_line = re.search(pattern, text)
794 if match_line is None and not ignore_fail:
795 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
796 elif match_line is not None:
797 old_line = match_line.group(0)
798 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
799 line = re.search(pattern, text).group(0)
800 log.debug("Updating %s to %s in %s", old_line, line, config_file)
801 with open(config_file, "wb") as file_handle:
802 file_handle.write(text)
805 def prep_cnf(config_files, params_dict, regex_dict=None):
807 Update all config files with the default overriding parameters,
808 i.e. override the values hard-coded in those config files.
810 :param config_files: config files to use for the configuration
811 :type config_files: [str]
812 :param params_dict: parameters to override the defaults in the config files
813 :type params_dict: {str, str}
814 :param regex_dict: regular expressions to use for matching the overriden parameters
815 :type regex_dict: {str, str} or None
816 :returns: list of prepared (modified) config paths
819 log.info("Preparing %s template config files", len(config_files))
821 src_config_paths = prep_config_paths(config_files)
822 new_config_paths = []
823 for config_path in src_config_paths:
824 new_config_path = generate_config_path(dumped=True)
825 shutil.copy(config_path, new_config_path)
826 new_config_paths.append(new_config_path)
828 for config_path in new_config_paths:
829 for param_key in params_dict.keys():
830 if regex_dict is None:
831 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
832 elif param_key in regex_dict.keys():
833 regex_val = regex_dict[param_key] % param_key.upper()
834 elif re.match("\w*_\d+$", param_key):
835 final_parameter, parent_id = \
836 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
837 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
838 % (parent_id, final_parameter.upper())
839 log.debug("Requested regex for %s is '%s'",
840 param_key, regex_val)
842 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
843 prep_cnf_value(config_path, params_dict[param_key],
844 regex=regex_val, ignore_fail=True)
845 log.info("Prepared template config file %s", config_path)
847 return new_config_paths
850 def generate_config_path(dumped=False):
852 Generate path for a temporary config name.
854 :param bool dumped: whether the file should be in the dump
855 directory or in temporary directory
856 :returns: generated config file path
859 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
860 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
873 def batch_update_cnf(cnf, vars):
875 Perform a batch update of multiple cnf variables.
877 :param cnf: CNF variable to update
878 :type cnf: BuildCnfVar object
879 :param vars: tuples of enumerated action and subtuple with data
880 :type vars: [(int, (str, int, str))]
881 :returns: updated CNF variable
882 :rtype: BuildCnfVar object
884 The actions are indexed in the same order: delete, update, add, child.
887 for (action, data) in vars:
890 last = cnf.update_cnf(var, ref, val)
893 last = cnf.add_cnf(var, ref, val)
894 elif action == Delete:
895 last = cnf.del_cnf(data)
896 elif action == Child: # only one depth supported
899 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
903 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
905 Build a CNF variable and save it in a config file.
907 :param str kind: name of the CNF variable
908 :param int instance: instance number of the CNF variable
909 :param vals: tuples of enumerated action and subtuple with data
910 :type vals: [(int, (str, int, str))]
911 :param str data: data for the CNF variable
912 :param filename: optional custom name of the config file
913 :type filename: str or None
914 :returns: name of the saved config file
917 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
918 batch_update_cnf(builder, vals)
919 filename = generate_config_path(dumped=True) if filename is None else filename
920 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
921 builder.save(filename)