1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Guest utility to wrap arnied related functionality through python calls.
27 Copyright: Intra2net AG
30 There are three types of setting some cnfvar configuration:
32 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed
33 config file without modifying its content in any way
34 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using
35 static file but rather as a template, replacing regex-matched values to
36 adapt it to different configurations
37 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries
38 and custom cnfvar classes and writing them into config files of a desired
39 format (json, cnf, or raw)
43 ------------------------------------------------------
55 log = logging.getLogger('pyi2ncommon.arnied_wrapper')
57 from .cnfline import build_cnfvar
63 #: default set_cnf binary
64 BIN_SET_CNF = "/usr/intranator/bin/set_cnf"
65 #: default location for template configuration files
67 #: default location for dumped configuration files
71 class ConfigError(Exception):
75 def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60):
77 Universal command run wrapper.
79 :param str cmd: command to run
80 :param bool ignore_errors: whether not to raise error on command failure
81 :param vm: vm to run on if running on a guest instead of the host
82 :type vm: VM object or None
83 :param int timeout: amount of seconds to wait for the program to run
84 :returns: command result output
86 :raises: :py:class:`OSError` if command failed and cannot be ignored
89 status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout)
90 stdout = stdout.encode()
96 raise subprocess.CalledProcessError(status, cmd, stderr=stderr)
97 return subprocess.CompletedProcess(cmd, status, stdout=stdout, stderr=stderr)
99 return subprocess.run(cmd, check=not ignore_errors, shell=True, capture_output=True)
102 def verify_running(process='arnied', timeout=60, vm=None):
104 Verify if a given process is running via 'pgrep'.
105 Normally this is used to check if arnied is running.
107 :param str process: process to verify if running
108 :param int timeout: run verification timeout
109 :param vm: vm to run on if running on a guest instead of the host
110 :type vm: VM object or None
111 :raises: :py:class:`RuntimeError` if process is not running
116 platform_str = " on %s" % vm.name
117 for i in range(timeout):
118 log.info("Checking whether %s is running%s (%i\%i)",
119 process, platform_str, i, timeout)
120 result = run_cmd(cmd="pgrep -l -x %s" % process,
121 ignore_errors=True, vm=vm)
122 if result.returncode == 0:
126 raise RuntimeError("Process %s does not seem to be running" % process)
129 # Basic functionality
132 def accept_licence(vm=None):
134 Accept the Intra2net license.
136 :param vm: vm to run on if running on a guest instead of the host
137 :type vm: VM object or None
139 This is mostly useful for simplified webpage access.
141 cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf'
142 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
144 cmd = "/usr/intranator/bin/arnied_helper --wait-for-program-end GENERATE"
145 run_cmd(cmd=cmd, vm=vm)
148 def go_online(provider_id, wait_online=True, timeout=60, vm=None):
150 Go online with the given provider id.
152 :param provider_id: provider to go online with
153 :type provider_id: int
154 :param wait_online: whether to wait until online
155 :type wait_online: bool
156 :param vm: vm to run on if running on a guest instead of the host
157 :type vm: VM object or None
159 .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online`
161 log.info("Switching to online mode with provider %d", provider_id)
163 get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm)
164 if b'PROVIDER,' not in get_cnf_res.stdout:
165 log.warning('There is no PROVIDER %d on the vm. Skipping go_online.',
169 cmd = 'tell-connd --online P%i' % provider_id
170 result = run_cmd(cmd=cmd, vm=vm)
174 wait_for_online(provider_id, timeout=timeout, vm=vm)
177 def go_offline(wait_offline=True, vm=None):
181 :param wait_offline: whether to wait until offline
182 :type wait_offline: bool
183 :param vm: vm to run on if running on a guest instead of the host
184 :type vm: VM object or None
186 .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline`
188 cmd = 'tell-connd --offline'
189 result = run_cmd(cmd=cmd, vm=vm)
193 if wait_offline is True:
194 wait_for_offline(vm=vm)
196 wait_for_offline(wait_offline, vm=vm)
199 def wait_for_offline(timeout=60, vm=None):
201 Wait for arnied to signal we are offline.
203 :param int timeout: maximum timeout for waiting
204 :param vm: vm to run on if running on a guest instead of the host
205 :type vm: VM object or None
207 _wait_for_online_status('offline', None, timeout, vm)
210 def wait_for_online(provider_id, timeout=60, vm=None):
212 Wait for arnied to signal we are online.
214 :param provider_id: provider to go online with
215 :type provider_id: int
216 :param int timeout: maximum timeout for waiting
217 :param vm: vm to run on if running on a guest instead of the host
218 :type vm: VM object or None
220 _wait_for_online_status('online', provider_id, timeout, vm)
223 def _wait_for_online_status(status, provider_id, timeout, vm):
224 # Don't use tell-connd --status here since the actual
225 # ONLINE signal to arnied is transmitted
226 # asynchronously via arnieclient_muxer.
228 if status == 'online':
229 expected_output = 'DEFAULT: 2'
230 set_status_func = lambda: go_online(provider_id, False, vm)
231 elif status == 'offline':
232 expected_output = 'DEFAULT: 0'
233 set_status_func = lambda: go_offline(False, vm)
235 raise ValueError('expect status "online" or "offline", not "{0}"!'
238 log.info("Waiting for arnied to be {0} within {1} seconds"
239 .format(status, timeout))
241 for i in range(timeout):
242 # arnied might invalidate the connd "connection barrier"
243 # after generate was running and switch to OFFLINE (race condition).
244 # -> tell arnied every ten seconds to go online again
245 if i % 10 == 0 and i != 0:
248 cmd = '/usr/intranator/bin/get_var ONLINE'
249 result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
252 if expected_output in result.stdout.decode():
253 log.info("arnied is {0}. Continuing.".format(status))
258 raise RuntimeError("We didn't manage to go {0} within {1} seconds\n"
259 .format(status, timeout))
262 def disable_virscan(vm=None):
264 Disable virscan that could block GENERATE and thus all configurations.
266 :param vm: vm to run on if running on a guest instead of the host
267 :type vm: VM object or None
269 log.info("Disabling virus database update")
270 unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm)
272 cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf"
273 result = run_cmd(cmd=cmd, vm=vm)
276 # TODO: this intervention should be solved in later arnied_helper tool
277 cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*"
278 result = run_cmd(cmd=cmd, vm=vm)
280 log.info("Virus database update disabled")
283 def email_transfer(vm=None):
285 Transfer all the emails using the guest tool arnied_helper.
287 :param vm: vm to run on if running on a guest instead of the host
288 :type vm: VM object or None
290 cmd = "/usr/intranator/bin/arnied_helper --transfer-mail"
291 result = run_cmd(cmd=cmd, vm=vm)
295 def wait_for_email_transfer(timeout=300, vm=None):
297 Wait until the mail queue is empty and all emails are sent.
299 :param int timeout: email transfer timeout
300 :param vm: vm to run on if running on a guest instead of the host
301 :type vm: VM object or None
303 for i in range(timeout):
305 # Retrigger mail queue in case something is deferred
306 # by an amavisd-new reconfiguration
307 run_cmd(cmd='postqueue -f', vm=vm)
308 log.info('Waiting for SMTP queue to get empty (%i/%i s)',
310 if not run_cmd(cmd='postqueue -j', vm=vm).stdout:
311 log.debug('SMTP queue is empty')
314 log.warning('Timeout reached but SMTP queue still not empty after {} s'
318 def schedule(program, exec_time=0, optional_args="", vm=None):
320 Schedule a program to be executed at a given unix time stamp.
322 :param str program: program whose execution is scheduled
323 :param int exec_time: scheduled time of program's execution
324 :param str optional_args: optional command line arguments
325 :param vm: vm to run on if running on a guest instead of the host
326 :type vm: VM object or None
328 log.info("Scheduling %s to be executed at %i", program, exec_time)
329 schedule_dir = "/var/intranator/schedule"
330 # clean previous schedules of the same program
331 files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir)
332 for file_name in files:
333 if file_name.startswith(program.upper()):
334 log.debug("Removing previous scheduled %s", file_name)
336 vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name))
338 os.unlink(os.path.join(schedule_dir, file_name))
340 contents = "%i\n%s\n" % (exec_time, optional_args)
342 tmp_file = tempfile.NamedTemporaryFile(mode="w+",
343 prefix=program.upper() + "_",
346 log.debug("Created temporary file %s", tmp_file.name)
347 tmp_file.write(contents)
349 moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name))
352 vm.copy_files_to(tmp_file.name, moved_tmp_file)
353 os.remove(tmp_file.name)
355 shutil.move(tmp_file.name, moved_tmp_file)
357 log.debug("Moved temporary file to %s", moved_tmp_file)
360 def wait_for_run(program, timeout=300, retries=10, vm=None):
362 Wait for a program using the guest arnied_helper tool.
364 :param str program: scheduled or running program to wait for
365 :param int timeout: program run timeout
366 :param int retries: number of tries to verify that the program is scheduled or running
367 :param vm: vm to run on if running on a guest instead of the host
368 :type vm: VM object or None
370 log.info("Waiting for program %s to finish with timeout %i",
372 for i in range(retries):
373 cmd = "/usr/intranator/bin/arnied_helper --is-scheduled-or-running " \
375 check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm)
376 if check_scheduled.returncode == 0:
380 log.warning("The program %s was not scheduled and is not running", program)
382 cmd = "/usr/intranator/bin/arnied_helper --wait-for-program-end " \
383 + program.upper() + " --wait-for-program-timeout " + str(timeout)
384 # add one second to make sure arnied_helper is finished when we expire
385 result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1)
386 log.debug(result.stdout)
389 # Configuration functionality
391 def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None):
393 Query arnied for a `cnf_key` and extract some information via regex.
395 :param str cnf_key: queried cnf key
396 :param int cnf_index: index of the cnf key
397 :param str regex: regex to apply on the queried cnf key data
398 :param bool compact: whether to retrieve compact version of the matched cnf keys
399 :param int timeout: arnied run verification timeout
400 :param vm: vm to run on if running on a guest instead of the host
401 :type vm: VM object or None
402 :returns: extracted information via the regex
405 If `cnf_index` is set to -1, retrieve and perform regex matching on all instances.
407 verify_running(timeout=timeout, vm=vm)
410 platform_str = " from %s" % vm.name
411 log.info("Extracting arnied value %s for %s%s using pattern %s",
412 cnf_index, cnf_key, platform_str, regex)
413 cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key,
414 " %s" % cnf_index if cnf_index != -1 else "")
415 output = run_cmd(cmd=cmd, vm=vm).stdout.decode()
416 return re.search(regex, output, flags=re.DOTALL)
419 def get_cnf_id(cnf_key, value, timeout=30, vm=None):
421 Get the id of a configuration of type `cnf_key` and name `value`.
423 :param str cnf_key: queried cnf key
424 :param str value: cnf value of the cnf key
425 :param int timeout: arnied run verification timeout
426 :param vm: vm to run on if running on a guest instead of the host
427 :type vm: VM object or None
428 :returns: the cnf id or -1 if no such cnf variable
431 verify_running(timeout=timeout, vm=vm)
432 regex = "%s,(\d+): \"%s\"" % (cnf_key, value)
433 cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm)
437 cnf_id = int(cnf_id.group(1))
438 log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id)
442 def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
444 Invoke get_cnf and return a nested CNF structure.
446 :param str varname: "varname" field of the CNF_VAR to look up
447 :param instance: "instance" of that variable to return
449 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
450 :param int timeout: arnied run verification timeout
451 :param vm: vm to run on if running on a guest instead of the host
452 :type vm: VM object or None
453 :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
454 :rtype: cnfvar option
456 verify_running(timeout=timeout, vm=vm)
457 # firstly, build argv for get_cnf
458 cmd = ["get_cnf", "-j"]
459 if varname is not None:
460 cmd.append("%s" % varname)
462 cmd.append("%d" % instance)
463 cmd_line = " ".join(cmd)
466 result = run_cmd(cmd=cmd_line, vm=vm)
467 (status, raw) = result.returncode, result.stdout
469 log.info("error %d executing \"%s\"", status, cmd_line)
473 # reading was successful, attempt to parse what we got
475 cnf = cnfvar.read_cnf_json(raw)
476 except TypeError as exn:
477 log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
479 except cnfvar.InvalidCNF as exn:
480 log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
484 return cnfvar.get_vars(cnf, data=data)
489 def get_cnfvar_id(varname, data, timeout=30, vm=None):
491 Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
493 :param str varname: "varname" field of the CNF_VAR to look up
494 :param str data: "data" field by which the resulting CNF_VAR list should be filtered
495 :param int timeout: arnied run verification timeout
496 :param vm: vm to run on if running on a guest instead of the host
497 :type vm: VM object or None
498 :returns: the cnf id or -1 if no such cnf variable
501 verify_running(timeout=timeout, vm=vm)
502 log.info("Extracting from arnied CNF_VAR %s with data %s",
504 cnf = get_cnfvar(varname=varname, data=data, vm=vm)
505 variables = cnf["cnf"]
506 if len(variables) == 0:
507 log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
510 first_instance = int(variables[0]["instance"])
511 log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
512 len(variables), first_instance)
513 return first_instance
516 def wait_for_generate(timeout=300, vm=None):
518 Wait for the 'generate' program to complete.
520 Arguments are similar to the ones from :py:method:`wait_for_run`.
522 wait_for_run('generate', timeout=timeout, retries=1, vm=vm)
523 wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm)
526 def unset_cnf(varname="", instance="", timeout=30, vm=None):
528 Remove configuration from arnied.
530 :param str varname: "varname" field of the CNF_VAR to unset
531 :param int instance: "instance" of that variable to unset
532 :param int timeout: arnied run verification timeout
533 :param vm: vm to run on if running on a guest instead of the host
534 :type vm: VM object or None
536 verify_running(timeout=timeout, vm=vm)
538 cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance)
539 run_cmd(cmd=cmd, vm=vm)
541 wait_for_generate(vm=vm)
544 def set_cnf(config_files, kind="cnf", timeout=30, vm=None):
546 Perform static arnied configuration through a set of config files.
548 :param config_files: config files to use for the configuration
549 :type config_files: [str]
550 :param str kind: "json" or "cnf"
551 :param int timeout: arnied run verification timeout
552 :param vm: vm to run on if running on a guest instead of the host
553 :type vm: VM object or None
554 :raises: :py:class:`ConfigError` if cannot apply file
556 The config files must be provided and are always expected to be found on
557 the host. If these are absolute paths, they will be kept as is or
558 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
559 the config files will be copied there as temporary files before applying.
561 log.info("Setting arnied configuration")
562 verify_running(timeout=timeout, vm=vm)
564 config_paths = prep_config_paths(config_files)
565 for config_path in config_paths:
566 with open(config_path, "rt", errors='replace') as config:
567 log.debug("Contents of applied %s:\n%s", config_path, config.read())
569 new_config_path = generate_config_path()
570 vm.copy_files_to(config_path, new_config_path)
571 config_path = new_config_path
572 argv = ["set_cnf", kind == "json" and "-j" or "", config_path]
574 result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm)
575 logging.debug(result)
576 if result.returncode != 0:
577 raise ConfigError("Failed to apply config %s%s, set_cnf returned %d"
579 " on %s" % vm.name if vm is not None else "",
583 wait_for_generate(vm=vm)
584 except Exception as ex:
585 # handle cases of remote configuration that leads to connection meltdown
586 if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError):
587 log.info("Resetting connection to %s", vm.name)
588 vm.session = vm.wait_for_login(timeout=10)
589 log.debug("Connection reset via remote error: %s", ex)
594 def set_cnf_semidynamic(config_files, params_dict, regex_dict=None,
595 kind="cnf", timeout=30, vm=None):
597 Perform semi-dynamic arnied configuration from an updated version of the
600 :param config_files: config files to use for the configuration
601 :type config_files: [str]
602 :param params_dict: parameters to override the defaults in the config files
603 :type params_dict: {str, str}
604 :param regex_dict: regular expressions to use for matching the overriden parameters
605 :type regex_dict: {str, str} or None
606 :param str kind: "json" or "cnf"
607 :param int timeout: arnied run verification timeout
608 :param vm: vm to run on if running on a guest instead of the host
609 :type vm: VM object or None
611 The config files must be provided and are always expected to be found on
612 the host. If these are absolute paths, they will be kept as is or
613 otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
614 the config files will be copied there as temporary files before applying.
616 log.info("Performing semi-dynamic arnied configuration")
618 config_paths = prep_cnf(config_files, params_dict, regex_dict)
619 set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm)
621 log.info("Semi-dynamic arnied configuration successful!")
624 def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
626 Perform dynamic arnied configuration from fully generated config files.
628 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
629 :type cnf: {str, str}
630 :param config_file: optional user supplied filename
631 :type config_file: str or None
632 :param str kind: "json", "cnf", or "raw"
633 :param int timeout: arnied run verification timeout
634 :param vm: vm to run on if running on a guest instead of the host
635 :type vm: VM object or None
636 :raises: :py:class:`ValueError` if `kind` is not an acceptable value
637 :raises: :py:class:`ConfigError` if cannot apply file
639 The config file might not be provided in which case a temporary file will
640 be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
641 an absolute path. If a vm is provided, the config file will be copied there
642 as a temporary file before applying.
644 if config_file is None:
645 config_path = generate_config_path(dumped=True)
646 elif os.path.isabs(config_file):
647 config_path = config_file
649 config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
650 generated = config_file is None
651 config_file = os.path.basename(config_path)
652 log.info("Using %s cnf file %s%s",
653 "generated" if generated else "user-supplied",
654 config_file, " on %s" % vm.name if vm is not None else "")
656 # Important to write bytes here to ensure text is encoded with latin-1
657 fd = open(config_path, "wb")
660 "raw": cnfvar.write_cnf_raw,
661 "json": cnfvar.write_cnf_json,
662 "cnf": cnfvar.write_cnf
664 SET_CNF_METHODS[kind](cnf, out=fd)
666 raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
670 log.info("Generated config file %s", config_path)
672 kind = "cnf" if kind != "json" else kind
673 set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
676 def set_cnf_pipe(cnf, timeout=30, block=False):
678 Set local configuration by talking to arnied via ``set_cnf``.
680 :param cnf: one key with the same value as *kind* and a list of cnfvars as value
681 :type cnf: {str, str}
682 :param int timeout: arnied run verification timeout
683 :param bool block: whether to wait for generate to complete the
685 :returns: whether ``set_cnf`` succeeded or not
688 This is obviously not generic but supposed to be run on the guest.
690 log.info("Setting arnied configuration through local pipe")
691 verify_running(timeout=timeout)
693 st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf))
696 log.error("Error applying configuration; status=%r" % exit)
697 log.error("and stderr:\n%s" % out)
699 log.debug("Configuration successfully passed to set_cnf, "
700 "read %d B from pipe" % len(out))
703 log.debug("Waiting for config job to complete")
706 log.debug("Exiting sucessfully")
710 def prep_config_paths(config_files, config_dir=None):
712 Prepare absolute paths for all configs at an expected location.
714 :param config_files: config files to use for the configuration
715 :type config_files: [str]
716 :param config_dir: config directory to prepend to the filepaths
717 :type config_dir: str or None
718 :returns: list of the full config paths
721 if config_dir is None:
722 config_dir = SRC_CONFIG_DIR
724 for config_file in config_files:
725 if os.path.isabs(config_file):
726 # Absolute path: The user requested a specific file
727 # f.e. needed for dynamic arnied config update
728 config_path = config_file
730 config_path = os.path.join(os.path.abspath(config_dir),
732 logging.debug("Using %s for original path %s", config_path, config_file)
733 config_paths.append(config_path)
737 def prep_cnf_value(config_file, value,
738 regex=None, template_key=None, ignore_fail=False):
740 Replace value in a provided arnied config file.
742 :param str config_file: file to use for the replacement
743 :param str value: value to replace the first matched group with
744 :param regex: regular expression to use when replacing a cnf value
745 :type regex: str or None
746 :param template_key: key of a quick template to use for the regex
747 :type template_key: str or None
748 :param bool ignore_fail: whether to ignore regex mismatching
749 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
751 In order to ensure better matching capabilities you are supposed to
752 provide a regex pattern with at least one subgroup to match your value.
753 What this means is that the value you like to replace is not directly
754 searched into the config text but matched within a larger regex in
755 in order to avoid any mismatch.
758 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
760 if template_key is None:
761 pattern = regex.encode()
763 samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"',
764 "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'}
765 pattern = samples[template_key].encode()
767 with open(config_file, "rb") as file_handle:
768 text = file_handle.read()
769 match_line = re.search(pattern, text)
771 if match_line is None and not ignore_fail:
772 raise ValueError("Pattern %s not found in %s" % (pattern, config_file))
773 elif match_line is not None:
774 old_line = match_line.group(0)
775 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
776 line = re.search(pattern, text).group(0)
777 log.debug("Updating %s to %s in %s", old_line, line, config_file)
778 with open(config_file, "wb") as file_handle:
779 file_handle.write(text)
782 def prep_cnf(config_files, params_dict, regex_dict=None):
784 Update all config files with the default overriding parameters,
785 i.e. override the values hard-coded in those config files.
787 :param config_files: config files to use for the configuration
788 :type config_files: [str]
789 :param params_dict: parameters to override the defaults in the config files
790 :type params_dict: {str, str}
791 :param regex_dict: regular expressions to use for matching the overriden parameters
792 :type regex_dict: {str, str} or None
793 :returns: list of prepared (modified) config paths
796 log.info("Preparing %s template config files", len(config_files))
798 src_config_paths = prep_config_paths(config_files)
799 new_config_paths = []
800 for config_path in src_config_paths:
801 new_config_path = generate_config_path(dumped=True)
802 shutil.copy(config_path, new_config_path)
803 new_config_paths.append(new_config_path)
805 for config_path in new_config_paths:
806 for param_key in params_dict.keys():
807 if regex_dict is None:
808 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
809 elif param_key in regex_dict.keys():
810 regex_val = regex_dict[param_key] % param_key.upper()
811 elif re.match("\w*_\d+$", param_key):
812 final_parameter, parent_id = \
813 re.match("(\w*)_(\d+)$", param_key).group(1, 2)
814 regex_val = "\(%s\) %s,\d+: \"(.*)\"" \
815 % (parent_id, final_parameter.upper())
816 log.debug("Requested regex for %s is '%s'",
817 param_key, regex_val)
819 regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper()
820 prep_cnf_value(config_path, params_dict[param_key],
821 regex=regex_val, ignore_fail=True)
822 log.info("Prepared template config file %s", config_path)
824 return new_config_paths
827 def generate_config_path(dumped=False):
829 Generate path for a temporary config name.
831 :param bool dumped: whether the file should be in the dump
832 directory or in temporary directory
833 :returns: generated config file path
836 dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None
837 fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir)
850 def batch_update_cnf(cnf, vars):
852 Perform a batch update of multiple cnf variables.
854 :param cnf: CNF variable to update
855 :type cnf: BuildCnfVar object
856 :param vars: tuples of enumerated action and subtuple with data
857 :type vars: [(int, (str, int, str))]
858 :returns: updated CNF variable
859 :rtype: BuildCnfVar object
861 The actions are indexed in the same order: delete, update, add, child.
864 for (action, data) in vars:
867 last = cnf.update_cnf(var, ref, val)
870 last = cnf.add_cnf(var, ref, val)
871 elif action == Delete:
872 last = cnf.del_cnf(data)
873 elif action == Child: # only one depth supported
876 cnf.add_cnf(var, ref, val, different_parent_line_no=last)
880 def build_cnf(kind, instance=0, vals=[], data="", filename=None):
882 Build a CNF variable and save it in a config file.
884 :param str kind: name of the CNF variable
885 :param int instance: instance number of the CNF variable
886 :param vals: tuples of enumerated action and subtuple with data
887 :type vals: [(int, (str, int, str))]
888 :param str data: data for the CNF variable
889 :param filename: optional custom name of the config file
890 :type filename: str or None
891 :returns: name of the saved config file
894 builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data)
895 batch_update_cnf(builder, vals)
896 filename = generate_config_path(dumped=True) if filename is None else filename
897 [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR)
898 builder.save(filename)