1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 DEPRECATED! Please do not extend this or add new uses of this module, use
28 :py:mod:`pyi2ncommon.arnied_api` or :py:mod:`pyi2ncommon.cnfvar` instead!
30 Copyright: Intra2net AG
32 There are three types of setting some cnfvar configuration:
34 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
35 config file without modifying its content in any way
36 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
37 static file but rather as a template, replacing regex-matched values to
38 adapt it to different configurations
39 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
40 and custom cnfvar classes and writing them into config files of a desired
41 format (json, cnf, or raw)
45 ------------------------------------------------------
57 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
59 from .cnfline import build_cnfvar
60 from . import cnfvar_old
65 #: default set_cnf binary
66 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
67 #: default arnied_helper binary
68 BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper"
69 #: default location for template configuration files
71 #: default location for dumped configuration files
75 class ConfigError(Exception):
79 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
81 Universal command run wrapper.
83 :param str cmd: command to run
84 :param bool ignore_errors: whether not to raise error on command failure
85 :param vm: vm to run on if running on a guest instead of the host
86 :type vm: :py:class:`virttest.qemu_vm.VM` or None
87 :param int timeout: amount of seconds to wait for the program to run
88 :returns: command result output where output (stdout/stderr) is bytes
89 (encoding dependent on environment and command given)
90 :rtype: :py:class:`subprocess.CompletedProcess`
91 :raises: :py:class:`OSError` if command failed and cannot be ignored
94 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
95 stdout = stdout.encode()
100 if not ignore_errors:
101 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
102 return subprocess.CompletedProcess(cmd, status,
103 stdout=stdout, stderr=stderr)
105 return subprocess.run(cmd, check=not ignore_errors, shell=True,
109 def verify_running(process='arnied', timeout=60, vm=None):
111 Verify if a given process is running via 'pgrep'.
113 :param str process: process to verify if running
114 :param int timeout: run verification timeout
115 :param vm: vm to run on if running on a guest instead of the host
116 :type vm: :py:class:`virttest.qemu_vm.VM` or None
117 :raises: :py:class:`RuntimeError` if process is not running
122 platform_str = " on %s" % vm.name
123 for i in range(timeout):
124 log.info("Checking whether %s is running%s (%i\%i)",
125 process, platform_str, i, timeout)
126 result = run_cmd(cmd="pgrep -l -x %s" % process,
127 ignore_errors=True, vm=vm)
128 if result.returncode == 0:
132 raise RuntimeError("Process %s does not seem to be running" % process)
135 # Basic functionality
138 def accept_licence(vm=None):
140 Accept the Intra2net license.
142 :param vm: vm to run on if running on a guest instead of the host
143 :type vm: :py:class:`virttest.qemu_vm.VM` or None
145 This is mostly useful for simplified webpage access.
147 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
148 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
150 wait_for_generate(vm=vm)
153 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
155 Go online with the given provider id.
157 :param provider_id: provider to go online with
158 :type provider_id: int
159 :param wait_online: whether to wait until online
160 :type wait_online: bool
161 :param int timeout: Seconds to wait in :py:func:`wait_for_online`
162 :param vm: vm to run on if running on a guest instead of the host
163 :type vm: :py:class:`virttest.qemu_vm.VM` or None
165 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
167 log.info("Switching to online mode with provider %d", provider_id)
169 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
170 if b'PROVIDER,' not in get_cnf_res.stdout:
171 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
175 cmd = 'tell-connd --online P%i' % provider_id
176 result = run_cmd(cmd=cmd, vm=vm)
180 wait_for_online(provider_id, timeout=timeout, vm=vm)
183 def go_offline(wait_offline=True, vm=None):
187 :param wait_offline: whether to wait until offline
188 :type wait_offline: bool
189 :param vm: vm to run on if running on a guest instead of the host
190 :type vm: :py:class:`virttest.qemu_vm.VM` or None
192 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
194 cmd = 'tell-connd --offline'
195 result = run_cmd(cmd=cmd, vm=vm)
199 if wait_offline is True:
200 wait_for_offline(vm=vm)
202 wait_for_offline(wait_offline, vm=vm)
205 def wait_for_offline(timeout=60, vm=None):
207 Wait for arnied to signal we are offline.
209 :param int timeout: maximum timeout for waiting
210 :param vm: vm to run on if running on a guest instead of the host
211 :type vm: :py:class:`virttest.qemu_vm.VM` or None
213 _wait_for_online_status('offline', None, timeout, vm)
216 def wait_for_online(provider_id, timeout=60, vm=None):
218 Wait for arnied to signal we are online.
220 :param provider_id: provider to go online with
221 :type provider_id: int
222 :param int timeout: maximum timeout for waiting
223 :param vm: vm to run on if running on a guest instead of the host
224 :type vm: :py:class:`virttest.qemu_vm.VM` or None
226 _wait_for_online_status('online', provider_id, timeout, vm)
229 def _wait_for_online_status(status, provider_id, timeout, vm):
230 # Don't use tell-connd --status here since the actual
231 # ONLINE signal to arnied is transmitted
232 # asynchronously via arnieclient_muxer.
234 if status == 'online':
235 expected_output = 'DEFAULT: 2'
236 set_status_func = lambda: go_online(provider_id, False, vm)
237 elif status == 'offline':
238 expected_output = 'DEFAULT: 0'
239 set_status_func = lambda: go_offline(False, vm)
241 raise ValueError('expect status "online" or "offline", not "{0}"!'
244 log.info("Waiting for arnied to be {0} within {1} seconds"
245 .format(status, timeout))
247 for i in range(timeout):
248 # arnied might invalidate the connd "connection barrier"
249 # after generate was running and switch to OFFLINE (race condition).
250 # -> tell arnied every ten seconds to go online again
251 if i % 10 == 0 and i != 0:
254 cmd = '/usr/intranator/bin/get_var ONLINE'
255 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
258 if expected_output in result.stdout.decode():
259 log.info("arnied is {0}. Continuing.".format(status))
264 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
265 .format(status, timeout))
268 def disable_virscan(vm=None):
270 Disable virscan that could block GENERATE and thus all configurations.
272 :param vm: vm to run on if running on a guest instead of the host
273 :type vm: :py:class:`virttest.qemu_vm.VM` or None
275 log.info("Disabling virus database update")
276 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
278 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
279 result = run_cmd(cmd=cmd, vm=vm)
282 # TODO: this intervention should be solved in later arnied_helper tool
283 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
284 result = run_cmd(cmd=cmd, vm=vm)
286 log.info("Virus database update disabled")
289 def email_transfer(vm=None):
291 Transfer all the emails using the guest tool arnied_helper.
293 :param vm: vm to run on if running on a guest instead of the host
294 :type vm: :py:class:`virttest.qemu_vm.VM` or None
296 cmd = f"{BIN_ARNIED_HELPER} --transfer-mail"
297 result = run_cmd(cmd=cmd, vm=vm)
301 def wait_for_email_transfer(timeout=300, vm=None):
303 Wait until the mail queue is empty and all emails are sent.
305 :param int timeout: email transfer timeout
306 :param vm: vm to run on if running on a guest instead of the host
307 :type vm: :py:class:`virttest.qemu_vm.VM` or None
309 for i in range(timeout):
311 # Retrigger mail queue in case something is deferred
312 # by an amavisd-new reconfiguration
313 run_cmd(cmd='postqueue -f', vm=vm)
314 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
316 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
317 log.debug('SMTP queue is empty')
320 log.warning('Timeout reached but SMTP queue still not empty after {} s'
324 def schedule(program, exec_time=0, optional_args="", vm=None):
326 Schedule a program to be executed at a given unix time stamp.
328 :param str program: program whose execution is scheduled
329 :param int exec_time: scheduled time of program's execution
330 :param str optional_args: optional command line arguments
331 :param vm: vm to run on if running on a guest instead of the host
332 :type vm: :py:class:`virttest.qemu_vm.VM` or None
334 log.info("Scheduling %s to be executed at %i", program, exec_time)
335 schedule_dir = "/var/intranator/schedule"
336 # clean previous schedules of the same program
337 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
338 for file_name in files:
339 if file_name.startswith(program.upper()):
340 log.debug("Removing previous scheduled %s", file_name)
342 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
344 os.unlink(os.path.join(schedule_dir, file_name))
346 contents = "%i\n%s\n" % (exec_time, optional_args)
348 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
349 prefix=program.upper() + "_",
352 log.debug("Created temporary file %s", tmp_file.name)
353 tmp_file.write(contents)
355 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
358 vm.copy_files_to(tmp_file.name, moved_tmp_file)
359 os.remove(tmp_file.name)
361 shutil.move(tmp_file.name, moved_tmp_file)
363 log.debug("Moved temporary file to %s", moved_tmp_file)
366 def wait_for_run(program, timeout=300, retries=10, vm=None):
368 Wait for a program using the guest arnied_helper tool.
370 :param str program: scheduled or running program to wait for
371 :param int timeout: program run timeout
372 :param int retries: number of tries to verify that the program is scheduled or running
373 :param vm: vm to run on if running on a guest instead of the host
374 :type vm: :py:class:`virttest.qemu_vm.VM` or None
376 log.info("Waiting for program %s to finish with timeout %i",
378 for i in range(retries):
379 cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \
381 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
382 if check_scheduled.returncode == 0:
386 log.warning("The program %s was not scheduled and is not running", program)
388 cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \
389 f"{program.upper()} --wait-for-program-timeout {timeout}"
390 # add one second to make sure arnied_helper is finished when we expire
391 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
392 log.debug(result.stdout)
395 def wait_for_arnied(timeout=60, vm=None):
397 Wait for arnied socket to be ready.
399 :param int timeout: maximum number of seconds to wait
400 :param vm: vm to run on if running on a guest instead of the host
401 :type vm: :py:class:`virttest.qemu_vm.VM` or None
403 cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \
404 f"--wait-for-arnied-socket-timeout {timeout}"
405 # add one second to make sure arnied_helper is finished when we expire
406 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
407 log.debug(result.stdout)
410 # Configuration functionality
412 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
414 Query arnied for a `cnf_key` and extract some information via regex.
416 :param str cnf_key: queried cnf key
417 :param int cnf_index: index of the cnf key
418 :param str regex: regex to apply on the queried cnf key data
419 :param bool compact: whether to retrieve compact version of the matched cnf keys
420 :param int timeout: arnied run verification timeout
421 :param vm: vm to run on if running on a guest instead of the host
422 :type vm: :py:class:`virttest.qemu_vm.VM` or None
423 :returns: extracted information via the regex
426 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
428 wait_for_arnied(timeout=timeout, vm=vm)
431 platform_str = " from %s" % vm.name
432 log.info("Extracting arnied value %s for %s%s using pattern %s",
433 cnf_index, cnf_key, platform_str, regex)
434 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
435 " %s" % cnf_index if cnf_index != -1 else "")
436 # get_cnf creates latin1-encoded output, transfer from VM removes non-ascii
437 output = run_cmd(cmd=cmd, vm=vm).stdout.decode('latin1')
438 return re.search(regex, output, flags=re.DOTALL)
441 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
443 Get the id of a configuration of type `cnf_key` and name `value`.
445 :param str cnf_key: queried cnf key
446 :param str value: cnf value of the cnf key
447 :param int timeout: arnied run verification timeout
448 :param vm: vm to run on if running on a guest instead of the host
449 :type vm: :py:class:`virttest.qemu_vm.VM` or None
450 :returns: the cnf id or -1 if no such cnf variable
453 wait_for_arnied(timeout=timeout, vm=vm)
454 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
455 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
459 cnf_id = int(cnf_id.group(1))
460 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
464 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
466 Invoke get_cnf and return a nested CNF structure.
468 :param str varname: "varname" field of the CNF_VAR to look up
469 :param instance: "instance" of that variable to return
471 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
472 :param int timeout: arnied run verification timeout
473 :param vm: vm to run on if running on a guest instead of the host
474 :type vm: :py:class:`virttest.qemu_vm.VM` or None
475 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
476 :rtype: cnfvar option
478 wait_for_arnied(timeout=timeout, vm=vm)
479 # firstly, build argv for get_cnf
480 cmd = ["get_cnf", "-j"]
481 if varname is not None:
482 cmd.append("%s" % varname)
484 cmd.append("%d" % instance)
485 cmd_line = " ".join(cmd)
488 result = run_cmd(cmd=cmd_line, vm=vm)
489 (status, raw) = result.returncode, result.stdout
491 log.info("error %d executing \"%s\"", status, cmd_line)
495 # reading was successful, attempt to parse what we got
497 # The output from "get_cnf -j" is already utf-8. This contrast with
498 # the output of "get_cnf" (no json) which is latin1.
499 if isinstance(raw, bytes):
500 raw = raw.decode("utf-8")
501 cnf = cnfvar_old.read_cnf_json(raw)
502 except TypeError as exn:
503 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
505 except cnfvar_old.InvalidCNF as exn:
506 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
510 return cnfvar_old.get_vars(cnf, data=data)
515 def get_cnfvar_id(varname, data, timeout=30, vm=None):
517 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
519 :param str varname: "varname" field of the CNF_VAR to look up
520 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
521 :param int timeout: arnied run verification timeout
522 :param vm: vm to run on if running on a guest instead of the host
523 :type vm: :py:class:`virttest.qemu_vm.VM` or None
524 :returns: the cnf id or -1 if no such cnf variable
527 wait_for_arnied(timeout=timeout, vm=vm)
528 log.info("Extracting from arnied CNF_VAR %s with data %s",
530 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
531 variables = cnf["cnf"]
532 if len(variables) == 0:
533 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
536 first_instance = int(variables[0]["instance"])
537 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
538 len(variables), first_instance)
539 return first_instance
542 def wait_for_generate(timeout=300, vm=None):
544 Wait for the 'generate' program to complete.
546 Arguments are similar to the ones from :py:method:`wait_for_run`.
548 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
549 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
552 def unset_cnf(varname="", instance="", timeout=30, vm=None):
554 Remove configuration from arnied.
556 :param str varname: "varname" field of the CNF_VAR to unset
557 :param int instance: "instance" of that variable to unset
558 :param int timeout: arnied run verification timeout
559 :param vm: vm to run on if running on a guest instead of the host
560 :type vm: :py:class:`virttest.qemu_vm.VM` or None
562 wait_for_arnied(timeout=timeout, vm=vm)
564 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
565 run_cmd(cmd=cmd, vm=vm)
567 wait_for_generate(vm=vm)
570 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
572 Perform static arnied configuration through a set of config files.
574 :param config_files: config files to use for the configuration
575 :type config_files: [str]
576 :param str kind: "json" or "cnf"
577 :param int timeout: arnied run verification timeout
578 :param vm: vm to run on if running on a guest instead of the host
579 :type vm: :py:class:`virttest.qemu_vm.VM` or None
580 :raises: :py:class:`ConfigError` if cannot apply file
582 The config files must be provided and are always expected to be found on
583 the host. If these are absolute paths, they will be kept as is or
584 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
585 the config files will be copied there as temporary files before applying.
587 log.info("Setting arnied configuration")
588 wait_for_arnied(timeout=timeout, vm=vm)
590 config_paths = prep_config_paths(config_files)
591 for config_path in config_paths:
592 with open(config_path, "rt", errors='replace') as config:
593 log.debug("Contents of applied %s:\n%s", config_path, config.read())
595 new_config_path = generate_config_path()
596 vm.copy_files_to(config_path, new_config_path)
597 config_path = new_config_path
598 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
600 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
601 logging.debug(result)
602 if result.returncode != 0:
603 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
605 " on %s" % vm.name if vm is not None else "",
609 wait_for_generate(vm=vm)
610 except Exception as ex:
611 # handle cases of remote configuration that leads to connection meltdown
612 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
613 log.info("Resetting connection to %s", vm.name)
614 vm.session = vm.wait_for_login(timeout=10)
615 log.debug("Connection reset via remote error: %s", ex)
620 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
621 kind="cnf", timeout=30, vm=None):
623 Perform semi-dynamic arnied configuration from an updated version of the
626 :param config_files: config files to use for the configuration
627 :type config_files: [str]
628 :param params_dict: parameters to override the defaults in the config files
629 :type params_dict: {str, str}
630 :param regex_dict: regular expressions to use for matching the overriden parameters
631 :type regex_dict: {str, str} or None
632 :param str kind: "json" or "cnf"
633 :param int timeout: arnied run verification timeout
634 :param vm: vm to run on if running on a guest instead of the host
635 :type vm: :py:class:`virttest.qemu_vm.VM` or None
637 The config files must be provided and are always expected to be found on
638 the host. If these are absolute paths, they will be kept as is or
639 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
640 the config files will be copied there as temporary files before applying.
642 log.info("Performing semi-dynamic arnied configuration")
644 config_paths = prep_cnf(config_files, params_dict, regex_dict)
645 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
647 log.info("Semi-dynamic arnied configuration successful!")
650 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
652 Perform dynamic arnied configuration from fully generated config files.
654 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
655 :type cnf: {str, str}
656 :param config_file: optional user supplied filename
657 :type config_file: str or None
658 :param str kind: "json", "cnf", or "raw"
659 :param int timeout: arnied run verification timeout
660 :param vm: vm to run on if running on a guest instead of the host
661 :type vm: :py:class:`virttest.qemu_vm.VM` or None
662 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
663 :raises: :py:class:`ConfigError` if cannot apply file
665 The config file might not be provided in which case a temporary file will
666 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
667 an absolute path. If a vm is provided, the config file will be copied there
668 as a temporary file before applying.
670 if config_file is None:
671 config_path = generate_config_path(dumped=True)
672 elif os.path.isabs(config_file):
673 config_path = config_file
675 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
676 generated = config_file is None
677 config_file = os.path.basename(config_path)
678 log.info("Using %s cnf file %s%s",
679 "generated" if generated else "user-supplied",
680 config_file, " on %s" % vm.name if vm is not None else "")
682 # Important to write bytes here to ensure text is encoded with latin-1
683 fd = open(config_path, "wb")
686 "raw": cnfvar_old.write_cnf_raw,
687 "json": cnfvar_old.write_cnf_json,
688 "cnf": cnfvar_old.write_cnf
690 SET_CNF_METHODS[kind](cnf, out=fd)
692 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
696 log.info("Generated config file %s", config_path)
698 kind = "cnf" if kind != "json" else kind
699 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
702 def set_cnf_pipe(cnf, timeout=30, block=False):
704 Set local configuration by talking to arnied via ``set_cnf``.
706 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
707 :type cnf: {str, str}
708 :param int timeout: arnied run verification timeout
709 :param bool block: whether to wait for generate to complete the
711 :returns: whether ``set_cnf`` succeeded or not
714 This is obviously not generic but supposed to be run on the guest.
716 log.info("Setting arnied configuration through local pipe")
717 wait_for_arnied(timeout=timeout)
719 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
722 log.error("Error applying configuration; status=%r" % exit)
723 log.error("and stderr:\n%s" % out)
725 log.debug("Configuration successfully passed to set_cnf, "
726 "read %d B from pipe" % len(out))
729 log.debug("Waiting for config job to complete")
732 log.debug("Exiting sucessfully")
736 def prep_config_paths(config_files, config_dir=None):
738 Prepare absolute paths for all configs at an expected location.
740 :param config_files: config files to use for the configuration
741 :type config_files: [str]
742 :param config_dir: config directory to prepend to the filepaths
743 :type config_dir: str or None
744 :returns: list of the full config paths
747 if config_dir is None:
748 config_dir = SRC_CONFIG_DIR
750 for config_file in config_files:
751 if os.path.isabs(config_file):
752 # Absolute path: The user requested a specific file
753 # f.e. needed for dynamic arnied config update
754 config_path = config_file
756 config_path = os.path.join(os.path.abspath(config_dir),
758 logging.debug("Using %s for original path %s", config_path, config_file)
759 config_paths.append(config_path)
763 def prep_cnf_value(config_file, value,
764 regex=None, template_key=None, ignore_fail=False):
766 Replace value in a provided arnied config file.
768 :param str config_file: file to use for the replacement
769 :param str value: value to replace the first matched group with
770 :param regex: regular expression to use when replacing a cnf value
771 :type regex: str or None
772 :param template_key: key of a quick template to use for the regex
773 :type template_key: str or None
774 :param bool ignore_fail: whether to ignore regex mismatching
775 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
777 In order to ensure better matching capabilities you are supposed to
778 provide a regex pattern with at least one subgroup to match your value.
779 What this means is that the value you like to replace is not directly
780 searched into the config text but matched within a larger regex in
781 in order to avoid any mismatch.
784 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
786 if template_key is None:
787 pattern = regex.encode()
789 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
790 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
791 pattern = samples[template_key].encode()
793 with open(config_file, "rb") as file_handle:
794 text = file_handle.read()
795 match_line = re.search(pattern, text)
797 if match_line is None and not ignore_fail:
798 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
799 elif match_line is not None:
800 old_line = match_line.group(0)
801 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
802 line = re.search(pattern, text).group(0)
803 log.debug("Updating %s to %s in %s", old_line, line, config_file)
804 with open(config_file, "wb") as file_handle:
805 file_handle.write(text)
808 def prep_cnf(config_files, params_dict, regex_dict=None):
810 Update all config files with the default overriding parameters,
811 i.e. override the values hard-coded in those config files.
813 :param config_files: config files to use for the configuration
814 :type config_files: [str]
815 :param params_dict: parameters to override the defaults in the config files
816 :type params_dict: {str, str}
817 :param regex_dict: regular expressions to use for matching the overriden parameters
818 :type regex_dict: {str, str} or None
819 :returns: list of prepared (modified) config paths
822 log.info("Preparing %s template config files", len(config_files))
824 src_config_paths = prep_config_paths(config_files)
825 new_config_paths = []
826 for config_path in src_config_paths:
827 new_config_path = generate_config_path(dumped=True)
828 shutil.copy(config_path, new_config_path)
829 new_config_paths.append(new_config_path)
831 for config_path in new_config_paths:
832 for param_key in params_dict.keys():
833 if regex_dict is None:
834 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
835 elif param_key in regex_dict.keys():
836 regex_val = regex_dict[param_key] % param_key.upper()
837 elif re.match("\w*_\d+$", param_key):
838 final_parameter, parent_id = \
839 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
840 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
841 % (parent_id, final_parameter.upper())
842 log.debug("Requested regex for %s is '%s'",
843 param_key, regex_val)
845 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
846 prep_cnf_value(config_path, params_dict[param_key],
847 regex=regex_val, ignore_fail=True)
848 log.info("Prepared template config file %s", config_path)
850 return new_config_paths
853 def generate_config_path(dumped=False):
855 Generate path for a temporary config name.
857 :param bool dumped: whether the file should be in the dump
858 directory or in temporary directory
859 :returns: generated config file path
862 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
863 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
876 def batch_update_cnf(cnf, vars):
878 Perform a batch update of multiple cnf variables.
880 :param cnf: CNF variable to update
881 :type cnf: BuildCnfVar object
882 :param vars: tuples of enumerated action and subtuple with data
883 :type vars: [(int, (str, int, str))]
884 :returns: updated CNF variable
885 :rtype: BuildCnfVar object
887 The actions are indexed in the same order: delete, update, add, child.
890 for (action, data) in vars:
893 last = cnf.update_cnf(var, ref, val)
896 last = cnf.add_cnf(var, ref, val)
897 elif action == Delete:
898 last = cnf.del_cnf(data)
899 elif action == Child: # only one depth supported
902 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
906 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
908 Build a CNF variable and save it in a config file.
910 :param str kind: name of the CNF variable
911 :param int instance: instance number of the CNF variable
912 :param vals: tuples of enumerated action and subtuple with data
913 :type vals: [(int, (str, int, str))]
914 :param str data: data for the CNF variable
915 :param filename: optional custom name of the config file
916 :type filename: str or None
917 :returns: name of the saved config file
920 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
921 batch_update_cnf(builder, vals)
922 filename = generate_config_path(dumped=True) if filename is None else filename
923 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
924 builder.save(filename)