Commit | Line | Data |
---|---|---|
11cbb815 PD |
1 | # The software in this package is distributed under the GNU General |
2 | # Public License version 2 (with a special exception described below). | |
3 | # | |
4 | # A copy of GNU General Public License (GPL) is included in this distribution, | |
5 | # in the file COPYING.GPL. | |
6 | # | |
7 | # As a special exception, if other files instantiate templates or use macros | |
8 | # or inline functions from this file, or you compile this file and link it | |
9 | # with other works to produce a work based on this file, this file | |
10 | # does not by itself cause the resulting work to be covered | |
11 | # by the GNU General Public License. | |
12 | # | |
13 | # However the source code for this file must still be made available | |
14 | # in accordance with section (3) of the GNU General Public License. | |
15 | # | |
16 | # This exception does not invalidate any other reasons why a work based | |
17 | # on this file might be covered by the GNU General Public License. | |
18 | # | |
19 | # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com> | |
20 | ||
f49f6323 PD |
21 | """ |
22 | ||
23 | SUMMARY | |
24 | ------------------------------------------------------ | |
25 | Guest utility to wrap arnied related functionality through python calls. | |
26 | ||
d7cdea2c CH |
27 | DEPRECATED! Please do not extend this or add new uses of this module, use |
28 | :py:mod:`pyi2ncommon.arnied_api` or :py:mod:`pyi2ncommon.cnfvar` instead! | |
f49f6323 | 29 | |
d7cdea2c | 30 | Copyright: Intra2net AG |
f49f6323 PD |
31 | |
32 | There are three types of setting some cnfvar configuration: | |
33 | ||
34 | 1) static (:py:class:`set_cnf`) - oldest method using a static preprocessed | |
35 | config file without modifying its content in any way | |
36 | 2) semi-dynamic (:py:class:`set_cnf_semidynamic`) - old method also using | |
37 | static file but rather as a template, replacing regex-matched values to | |
38 | adapt it to different configurations | |
39 | 3) dynamic (:py:class:`set_cnf_dynamic`) - new method using dictionaries | |
40 | and custom cnfvar classes and writing them into config files of a desired | |
41 | format (json, cnf, or raw) | |
42 | ||
f49f6323 PD |
43 | |
44 | INTERFACE | |
45 | ------------------------------------------------------ | |
46 | ||
47 | """ | |
48 | ||
49 | import os | |
50 | import sys | |
51 | import time | |
52 | import re | |
53 | import subprocess | |
54 | import shutil | |
55 | import tempfile | |
56 | import logging | |
3de8b4d8 | 57 | log = logging.getLogger('pyi2ncommon.arnied_wrapper') |
f49f6323 | 58 | |
30521dad | 59 | from .cnfline import build_cnfvar |
7abff5a7 | 60 | from . import cnfvar_old |
30521dad | 61 | from . import sysmisc |
7102665a | 62 | |
30521dad | 63 | |
f49f6323 PD |
64 | |
65 | #: default set_cnf binary | |
66 | BIN_SET_CNF = "/usr/intranator/bin/set_cnf" | |
f1ca3964 SA |
67 | #: default arnied_helper binary |
68 | BIN_ARNIED_HELPER = "/usr/intranator/bin/arnied_helper" | |
f49f6323 PD |
69 | #: default location for template configuration files |
70 | SRC_CONFIG_DIR = "." | |
71 | #: default location for dumped configuration files | |
72 | DUMP_CONFIG_DIR = "." | |
73 | ||
74 | ||
75 | class ConfigError(Exception): | |
76 | pass | |
77 | ||
78 | ||
dfa7c024 | 79 | def run_cmd(cmd="", ignore_errors=False, vm=None, timeout=60): |
f49f6323 PD |
80 | """ |
81 | Universal command run wrapper. | |
82 | ||
83 | :param str cmd: command to run | |
84 | :param bool ignore_errors: whether not to raise error on command failure | |
85 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 86 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
dfa7c024 | 87 | :param int timeout: amount of seconds to wait for the program to run |
2dd87078 CH |
88 | :returns: command result output where output (stdout/stderr) is bytes |
89 | (encoding dependent on environment and command given) | |
90 | :rtype: :py:class:`subprocess.CompletedProcess` | |
f49f6323 PD |
91 | :raises: :py:class:`OSError` if command failed and cannot be ignored |
92 | """ | |
93 | if vm is not None: | |
dfa7c024 | 94 | status, stdout = vm.session.cmd_status_output(cmd, timeout=timeout) |
f49f6323 PD |
95 | stdout = stdout.encode() |
96 | stderr = b"" | |
97 | if status != 0: | |
98 | stderr = stdout | |
99 | stdout = b"" | |
100 | if not ignore_errors: | |
101 | raise subprocess.CalledProcessError(status, cmd, stderr=stderr) | |
2dd87078 CH |
102 | return subprocess.CompletedProcess(cmd, status, |
103 | stdout=stdout, stderr=stderr) | |
f49f6323 | 104 | else: |
2dd87078 CH |
105 | return subprocess.run(cmd, check=not ignore_errors, shell=True, |
106 | capture_output=True) | |
f49f6323 PD |
107 | |
108 | ||
109 | def verify_running(process='arnied', timeout=60, vm=None): | |
110 | """ | |
111 | Verify if a given process is running via 'pgrep'. | |
f49f6323 PD |
112 | |
113 | :param str process: process to verify if running | |
114 | :param int timeout: run verification timeout | |
115 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 116 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
117 | :raises: :py:class:`RuntimeError` if process is not running |
118 | """ | |
119 | platform_str = "" | |
120 | if vm is not None: | |
121 | vm.verify_alive() | |
122 | platform_str = " on %s" % vm.name | |
123 | for i in range(timeout): | |
124 | log.info("Checking whether %s is running%s (%i\%i)", | |
125 | process, platform_str, i, timeout) | |
126 | result = run_cmd(cmd="pgrep -l -x %s" % process, | |
127 | ignore_errors=True, vm=vm) | |
128 | if result.returncode == 0: | |
129 | log.debug(result) | |
130 | return | |
131 | time.sleep(1) | |
132 | raise RuntimeError("Process %s does not seem to be running" % process) | |
133 | ||
134 | ||
135 | # Basic functionality | |
136 | ||
137 | ||
138 | def accept_licence(vm=None): | |
139 | """ | |
140 | Accept the Intra2net license. | |
141 | ||
142 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 143 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
144 | |
145 | This is mostly useful for simplified webpage access. | |
146 | """ | |
147 | cmd = 'echo "LICENSE_ACCEPTED,0: \\"1\\"" | set_cnf' | |
148 | result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm) | |
149 | log.debug(result) | |
1ac0318c | 150 | wait_for_generate(vm=vm) |
f49f6323 PD |
151 | |
152 | ||
153 | def go_online(provider_id, wait_online=True, timeout=60, vm=None): | |
154 | """ | |
155 | Go online with the given provider id. | |
156 | ||
157 | :param provider_id: provider to go online with | |
158 | :type provider_id: int | |
159 | :param wait_online: whether to wait until online | |
160 | :type wait_online: bool | |
7628bc48 | 161 | :param int timeout: Seconds to wait in :py:func:`wait_for_online` |
f49f6323 | 162 | :param vm: vm to run on if running on a guest instead of the host |
f4c8f40e | 163 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
164 | |
165 | .. seealso:: :py:func:`go_offline`, :py:func:`wait_for_online` | |
166 | """ | |
167 | log.info("Switching to online mode with provider %d", provider_id) | |
168 | ||
169 | get_cnf_res = run_cmd(cmd='get_cnf PROVIDER %d' % provider_id, vm=vm) | |
170 | if b'PROVIDER,' not in get_cnf_res.stdout: | |
33b0346a CH |
171 | log.warning('There is no PROVIDER %d on the vm. Skipping go_online.', |
172 | provider_id) | |
f49f6323 PD |
173 | return |
174 | ||
175 | cmd = 'tell-connd --online P%i' % provider_id | |
176 | result = run_cmd(cmd=cmd, vm=vm) | |
177 | log.debug(result) | |
178 | ||
179 | if wait_online: | |
180 | wait_for_online(provider_id, timeout=timeout, vm=vm) | |
181 | ||
182 | ||
183 | def go_offline(wait_offline=True, vm=None): | |
184 | """ | |
185 | Go offline. | |
186 | ||
187 | :param wait_offline: whether to wait until offline | |
188 | :type wait_offline: bool | |
189 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 190 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
191 | |
192 | .. seealso:: :py:func:`go_online`, :py:func:`wait_for_offline` | |
193 | """ | |
194 | cmd = 'tell-connd --offline' | |
195 | result = run_cmd(cmd=cmd, vm=vm) | |
196 | log.debug(result) | |
197 | ||
198 | if wait_offline: | |
199 | if wait_offline is True: | |
200 | wait_for_offline(vm=vm) | |
201 | else: | |
202 | wait_for_offline(wait_offline, vm=vm) | |
203 | ||
204 | ||
205 | def wait_for_offline(timeout=60, vm=None): | |
206 | """ | |
207 | Wait for arnied to signal we are offline. | |
208 | ||
209 | :param int timeout: maximum timeout for waiting | |
210 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 211 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
212 | """ |
213 | _wait_for_online_status('offline', None, timeout, vm) | |
214 | ||
215 | ||
216 | def wait_for_online(provider_id, timeout=60, vm=None): | |
217 | """ | |
218 | Wait for arnied to signal we are online. | |
219 | ||
220 | :param provider_id: provider to go online with | |
221 | :type provider_id: int | |
222 | :param int timeout: maximum timeout for waiting | |
223 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 224 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
225 | """ |
226 | _wait_for_online_status('online', provider_id, timeout, vm) | |
227 | ||
228 | ||
229 | def _wait_for_online_status(status, provider_id, timeout, vm): | |
230 | # Don't use tell-connd --status here since the actual | |
231 | # ONLINE signal to arnied is transmitted | |
232 | # asynchronously via arnieclient_muxer. | |
233 | ||
234 | if status == 'online': | |
235 | expected_output = 'DEFAULT: 2' | |
236 | set_status_func = lambda: go_online(provider_id, False, vm) | |
237 | elif status == 'offline': | |
238 | expected_output = 'DEFAULT: 0' | |
239 | set_status_func = lambda: go_offline(False, vm) | |
240 | else: | |
241 | raise ValueError('expect status "online" or "offline", not "{0}"!' | |
242 | .format(status)) | |
243 | ||
244 | log.info("Waiting for arnied to be {0} within {1} seconds" | |
245 | .format(status, timeout)) | |
246 | ||
247 | for i in range(timeout): | |
248 | # arnied might invalidate the connd "connection barrier" | |
249 | # after generate was running and switch to OFFLINE (race condition). | |
250 | # -> tell arnied every ten seconds to go online again | |
251 | if i % 10 == 0 and i != 0: | |
252 | set_status_func() | |
253 | ||
254 | cmd = '/usr/intranator/bin/get_var ONLINE' | |
255 | result = run_cmd(cmd=cmd, ignore_errors=True, vm=vm) | |
256 | log.debug(result) | |
257 | ||
258 | if expected_output in result.stdout.decode(): | |
259 | log.info("arnied is {0}. Continuing.".format(status)) | |
260 | return | |
261 | ||
262 | time.sleep(1) | |
263 | ||
264 | raise RuntimeError("We didn't manage to go {0} within {1} seconds\n" | |
265 | .format(status, timeout)) | |
266 | ||
267 | ||
268 | def disable_virscan(vm=None): | |
269 | """ | |
270 | Disable virscan that could block GENERATE and thus all configurations. | |
271 | ||
272 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 273 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
274 | """ |
275 | log.info("Disabling virus database update") | |
276 | unset_cnf("VIRSCAN_UPDATE_CRON", vm=vm) | |
277 | ||
278 | cmd = "echo 'VIRSCAN_UPDATE_DNS_PUSH,0:\"0\"' |set_cnf" | |
279 | result = run_cmd(cmd=cmd, vm=vm) | |
280 | log.debug(result) | |
281 | ||
282 | # TODO: this intervention should be solved in later arnied_helper tool | |
283 | cmd = "rm -f /var/intranator/schedule/UPDATE_VIRSCAN_NODIAL*" | |
284 | result = run_cmd(cmd=cmd, vm=vm) | |
285 | log.debug(result) | |
286 | log.info("Virus database update disabled") | |
287 | ||
288 | ||
289 | def email_transfer(vm=None): | |
290 | """ | |
291 | Transfer all the emails using the guest tool arnied_helper. | |
292 | ||
293 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 294 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 | 295 | """ |
f1ca3964 | 296 | cmd = f"{BIN_ARNIED_HELPER} --transfer-mail" |
f49f6323 PD |
297 | result = run_cmd(cmd=cmd, vm=vm) |
298 | log.debug(result) | |
299 | ||
300 | ||
301 | def wait_for_email_transfer(timeout=300, vm=None): | |
302 | """ | |
303 | Wait until the mail queue is empty and all emails are sent. | |
304 | ||
305 | :param int timeout: email transfer timeout | |
306 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 307 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
308 | """ |
309 | for i in range(timeout): | |
310 | if i % 10 == 0: | |
311 | # Retrigger mail queue in case something is deferred | |
312 | # by an amavisd-new reconfiguration | |
313 | run_cmd(cmd='postqueue -f', vm=vm) | |
12d603bf | 314 | log.info('Waiting for SMTP queue to get empty (%i/%i s)', |
f49f6323 | 315 | i, timeout) |
68d12e8a | 316 | if not run_cmd(cmd='postqueue -j', vm=vm).stdout: |
12d603bf CH |
317 | log.debug('SMTP queue is empty') |
318 | return | |
f49f6323 | 319 | time.sleep(1) |
12d603bf CH |
320 | log.warning('Timeout reached but SMTP queue still not empty after {} s' |
321 | .format(timeout)) | |
f49f6323 PD |
322 | |
323 | ||
324 | def schedule(program, exec_time=0, optional_args="", vm=None): | |
325 | """ | |
326 | Schedule a program to be executed at a given unix time stamp. | |
327 | ||
328 | :param str program: program whose execution is scheduled | |
329 | :param int exec_time: scheduled time of program's execution | |
330 | :param str optional_args: optional command line arguments | |
331 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 332 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
333 | """ |
334 | log.info("Scheduling %s to be executed at %i", program, exec_time) | |
f49f6323 | 335 | schedule_dir = "/var/intranator/schedule" |
ef5c0c20 | 336 | # clean previous schedules of the same program |
f49f6323 PD |
337 | files = vm.session.cmd("ls " + schedule_dir).split() if vm else os.listdir(schedule_dir) |
338 | for file_name in files: | |
339 | if file_name.startswith(program.upper()): | |
340 | log.debug("Removing previous scheduled %s", file_name) | |
341 | if vm: | |
ef5c0c20 | 342 | vm.session.cmd("rm -f " + os.path.join(schedule_dir, file_name)) |
f49f6323 PD |
343 | else: |
344 | os.unlink(os.path.join(schedule_dir, file_name)) | |
ef5c0c20 SA |
345 | |
346 | contents = "%i\n%s\n" % (exec_time, optional_args) | |
347 | ||
348 | tmp_file = tempfile.NamedTemporaryFile(mode="w+", | |
349 | prefix=program.upper() + "_", | |
350 | dir=DUMP_CONFIG_DIR, | |
351 | delete=False) | |
352 | log.debug("Created temporary file %s", tmp_file.name) | |
353 | tmp_file.write(contents) | |
354 | tmp_file.close() | |
355 | moved_tmp_file = os.path.join(schedule_dir, os.path.basename(tmp_file.name)) | |
356 | ||
f49f6323 | 357 | if vm: |
ef5c0c20 SA |
358 | vm.copy_files_to(tmp_file.name, moved_tmp_file) |
359 | os.remove(tmp_file.name) | |
f49f6323 PD |
360 | else: |
361 | shutil.move(tmp_file.name, moved_tmp_file) | |
ef5c0c20 | 362 | |
f49f6323 PD |
363 | log.debug("Moved temporary file to %s", moved_tmp_file) |
364 | ||
365 | ||
366 | def wait_for_run(program, timeout=300, retries=10, vm=None): | |
367 | """ | |
368 | Wait for a program using the guest arnied_helper tool. | |
369 | ||
370 | :param str program: scheduled or running program to wait for | |
371 | :param int timeout: program run timeout | |
372 | :param int retries: number of tries to verify that the program is scheduled or running | |
373 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 374 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
375 | """ |
376 | log.info("Waiting for program %s to finish with timeout %i", | |
377 | program, timeout) | |
378 | for i in range(retries): | |
f1ca3964 | 379 | cmd = f"{BIN_ARNIED_HELPER} --is-scheduled-or-running " \ |
f49f6323 PD |
380 | + program.upper() |
381 | check_scheduled = run_cmd(cmd=cmd, ignore_errors=True, vm=vm) | |
382 | if check_scheduled.returncode == 0: | |
383 | break | |
384 | time.sleep(1) | |
80d82f13 PD |
385 | else: |
386 | log.warning("The program %s was not scheduled and is not running", program) | |
387 | return | |
f1ca3964 SA |
388 | cmd = f"{BIN_ARNIED_HELPER} --wait-for-program-end " \ |
389 | f"{program.upper()} --wait-for-program-timeout {timeout}" | |
390 | # add one second to make sure arnied_helper is finished when we expire | |
391 | result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1) | |
392 | log.debug(result.stdout) | |
393 | ||
394 | ||
395 | def wait_for_arnied(timeout=60, vm=None): | |
396 | """ | |
397 | Wait for arnied socket to be ready. | |
398 | ||
399 | :param int timeout: maximum number of seconds to wait | |
400 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 401 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f1ca3964 SA |
402 | """ |
403 | cmd = f"{BIN_ARNIED_HELPER} --wait-for-arnied-socket " \ | |
404 | f"--wait-for-arnied-socket-timeout {timeout}" | |
3472b405 SA |
405 | # add one second to make sure arnied_helper is finished when we expire |
406 | result = run_cmd(cmd=cmd, vm=vm, timeout=timeout+1) | |
f49f6323 PD |
407 | log.debug(result.stdout) |
408 | ||
409 | ||
410 | # Configuration functionality | |
411 | ||
412 | def get_cnf(cnf_key, cnf_index=1, regex=".*", compact=False, timeout=30, vm=None): | |
413 | """ | |
414 | Query arnied for a `cnf_key` and extract some information via regex. | |
415 | ||
416 | :param str cnf_key: queried cnf key | |
417 | :param int cnf_index: index of the cnf key | |
418 | :param str regex: regex to apply on the queried cnf key data | |
419 | :param bool compact: whether to retrieve compact version of the matched cnf keys | |
420 | :param int timeout: arnied run verification timeout | |
421 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 422 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
423 | :returns: extracted information via the regex |
424 | :rtype: Match object | |
425 | ||
426 | If `cnf_index` is set to -1, retrieve and perform regex matching on all instances. | |
427 | """ | |
f1ca3964 | 428 | wait_for_arnied(timeout=timeout, vm=vm) |
f49f6323 PD |
429 | platform_str = "" |
430 | if vm is not None: | |
431 | platform_str = " from %s" % vm.name | |
432 | log.info("Extracting arnied value %s for %s%s using pattern %s", | |
433 | cnf_index, cnf_key, platform_str, regex) | |
434 | cmd = "get_cnf%s %s%s" % (" -c " if compact else "", cnf_key, | |
435 | " %s" % cnf_index if cnf_index != -1 else "") | |
6c691734 CH |
436 | # get_cnf creates latin1-encoded output, transfer from VM removes non-ascii |
437 | output = run_cmd(cmd=cmd, vm=vm).stdout.decode('latin1') | |
f49f6323 PD |
438 | return re.search(regex, output, flags=re.DOTALL) |
439 | ||
440 | ||
441 | def get_cnf_id(cnf_key, value, timeout=30, vm=None): | |
442 | """ | |
443 | Get the id of a configuration of type `cnf_key` and name `value`. | |
444 | ||
445 | :param str cnf_key: queried cnf key | |
446 | :param str value: cnf value of the cnf key | |
447 | :param int timeout: arnied run verification timeout | |
448 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 449 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
450 | :returns: the cnf id or -1 if no such cnf variable |
451 | :rtype: int | |
452 | """ | |
f1ca3964 | 453 | wait_for_arnied(timeout=timeout, vm=vm) |
f49f6323 PD |
454 | regex = "%s,(\d+): \"%s\"" % (cnf_key, value) |
455 | cnf_id = get_cnf(cnf_key, cnf_index=-1, regex=regex, compact=True, vm=vm) | |
456 | if cnf_id is None: | |
457 | cnf_id = -1 | |
458 | else: | |
459 | cnf_id = int(cnf_id.group(1)) | |
460 | log.info("Retrieved id \"%s\" for %s is %i", value, cnf_key, cnf_id) | |
461 | return cnf_id | |
462 | ||
463 | ||
464 | def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None): | |
465 | """ | |
466 | Invoke get_cnf and return a nested CNF structure. | |
467 | ||
468 | :param str varname: "varname" field of the CNF_VAR to look up | |
469 | :param instance: "instance" of that variable to return | |
470 | :type instance: int | |
471 | :param str data: "data" field by which the resulting CNF_VAR list should be filtered | |
472 | :param int timeout: arnied run verification timeout | |
473 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 474 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
475 | :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed |
476 | :rtype: cnfvar option | |
477 | """ | |
f1ca3964 | 478 | wait_for_arnied(timeout=timeout, vm=vm) |
f49f6323 PD |
479 | # firstly, build argv for get_cnf |
480 | cmd = ["get_cnf", "-j"] | |
481 | if varname is not None: | |
482 | cmd.append("%s" % varname) | |
483 | if instance: | |
484 | cmd.append("%d" % instance) | |
485 | cmd_line = " ".join(cmd) | |
486 | ||
487 | # now invoke get_cnf | |
488 | result = run_cmd(cmd=cmd_line, vm=vm) | |
489 | (status, raw) = result.returncode, result.stdout | |
490 | if status != 0: | |
491 | log.info("error %d executing \"%s\"", status, cmd_line) | |
492 | log.debug(raw) | |
493 | return None | |
494 | ||
495 | # reading was successful, attempt to parse what we got | |
496 | try: | |
35072357 JR |
497 | # The output from "get_cnf -j" is already utf-8. This contrast with |
498 | # the output of "get_cnf" (no json) which is latin1. | |
499 | if isinstance(raw, bytes): | |
500 | raw = raw.decode("utf-8") | |
7abff5a7 | 501 | cnf = cnfvar_old.read_cnf_json(raw) |
f49f6323 PD |
502 | except TypeError as exn: |
503 | log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line) | |
504 | return None | |
7abff5a7 | 505 | except cnfvar_old.InvalidCNF as exn: |
f49f6323 PD |
506 | log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line) |
507 | return None | |
508 | ||
509 | if data is not None: | |
7abff5a7 | 510 | return cnfvar_old.get_vars(cnf, data=data) |
f49f6323 PD |
511 | |
512 | return cnf | |
513 | ||
514 | ||
515 | def get_cnfvar_id(varname, data, timeout=30, vm=None): | |
516 | """ | |
517 | Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`. | |
518 | ||
519 | :param str varname: "varname" field of the CNF_VAR to look up | |
520 | :param str data: "data" field by which the resulting CNF_VAR list should be filtered | |
521 | :param int timeout: arnied run verification timeout | |
522 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 523 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
524 | :returns: the cnf id or -1 if no such cnf variable |
525 | :rtype: int | |
526 | """ | |
f1ca3964 | 527 | wait_for_arnied(timeout=timeout, vm=vm) |
f49f6323 PD |
528 | log.info("Extracting from arnied CNF_VAR %s with data %s", |
529 | varname, data) | |
530 | cnf = get_cnfvar(varname=varname, data=data, vm=vm) | |
531 | variables = cnf["cnf"] | |
532 | if len(variables) == 0: | |
533 | log.info("CNF_VAR extraction unsuccessful, defaulting to -1") | |
534 | # preserve behavior | |
535 | return -1 | |
536 | first_instance = int(variables[0]["instance"]) | |
537 | log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)", | |
538 | len(variables), first_instance) | |
539 | return first_instance | |
540 | ||
541 | ||
80d82f13 | 542 | def wait_for_generate(timeout=300, vm=None): |
f49f6323 PD |
543 | """ |
544 | Wait for the 'generate' program to complete. | |
545 | ||
80d82f13 | 546 | Arguments are similar to the ones from :py:method:`wait_for_run`. |
f49f6323 | 547 | """ |
80d82f13 PD |
548 | wait_for_run('generate', timeout=timeout, retries=1, vm=vm) |
549 | wait_for_run('generate_offline', timeout=timeout, retries=1, vm=vm) | |
f49f6323 PD |
550 | |
551 | ||
d9446155 | 552 | def unset_cnf(varname="", instance="", timeout=30, vm=None): |
f49f6323 PD |
553 | """ |
554 | Remove configuration from arnied. | |
555 | ||
556 | :param str varname: "varname" field of the CNF_VAR to unset | |
557 | :param int instance: "instance" of that variable to unset | |
d9446155 | 558 | :param int timeout: arnied run verification timeout |
f49f6323 | 559 | :param vm: vm to run on if running on a guest instead of the host |
f4c8f40e | 560 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 | 561 | """ |
f1ca3964 | 562 | wait_for_arnied(timeout=timeout, vm=vm) |
d9446155 | 563 | |
f49f6323 PD |
564 | cmd = "get_cnf %s %s | set_cnf -x" % (varname, instance) |
565 | run_cmd(cmd=cmd, vm=vm) | |
566 | ||
80d82f13 | 567 | wait_for_generate(vm=vm) |
f49f6323 PD |
568 | |
569 | ||
570 | def set_cnf(config_files, kind="cnf", timeout=30, vm=None): | |
571 | """ | |
572 | Perform static arnied configuration through a set of config files. | |
573 | ||
574 | :param config_files: config files to use for the configuration | |
575 | :type config_files: [str] | |
576 | :param str kind: "json" or "cnf" | |
577 | :param int timeout: arnied run verification timeout | |
578 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 579 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
580 | :raises: :py:class:`ConfigError` if cannot apply file |
581 | ||
582 | The config files must be provided and are always expected to be found on | |
583 | the host. If these are absolute paths, they will be kept as is or | |
584 | otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided, | |
585 | the config files will be copied there as temporary files before applying. | |
586 | """ | |
587 | log.info("Setting arnied configuration") | |
f1ca3964 | 588 | wait_for_arnied(timeout=timeout, vm=vm) |
f49f6323 PD |
589 | |
590 | config_paths = prep_config_paths(config_files) | |
591 | for config_path in config_paths: | |
592 | with open(config_path, "rt", errors='replace') as config: | |
593 | log.debug("Contents of applied %s:\n%s", config_path, config.read()) | |
594 | if vm is not None: | |
595 | new_config_path = generate_config_path() | |
596 | vm.copy_files_to(config_path, new_config_path) | |
597 | config_path = new_config_path | |
598 | argv = ["set_cnf", kind == "json" and "-j" or "", config_path] | |
599 | ||
600 | result = run_cmd(" ".join(argv), ignore_errors=True, vm=vm) | |
601 | logging.debug(result) | |
602 | if result.returncode != 0: | |
603 | raise ConfigError("Failed to apply config %s%s, set_cnf returned %d" | |
604 | % (config_path, | |
605 | " on %s" % vm.name if vm is not None else "", | |
606 | result.returncode)) | |
607 | ||
608 | try: | |
80d82f13 | 609 | wait_for_generate(vm=vm) |
f49f6323 PD |
610 | except Exception as ex: |
611 | # handle cases of remote configuration that leads to connection meltdown | |
612 | if vm is not None and isinstance(ex, sys.modules["aexpect"].ShellProcessTerminatedError): | |
613 | log.info("Resetting connection to %s", vm.name) | |
614 | vm.session = vm.wait_for_login(timeout=10) | |
615 | log.debug("Connection reset via remote error: %s", ex) | |
616 | else: | |
617 | raise ex | |
618 | ||
619 | ||
620 | def set_cnf_semidynamic(config_files, params_dict, regex_dict=None, | |
621 | kind="cnf", timeout=30, vm=None): | |
622 | """ | |
623 | Perform semi-dynamic arnied configuration from an updated version of the | |
624 | config files. | |
625 | ||
626 | :param config_files: config files to use for the configuration | |
627 | :type config_files: [str] | |
628 | :param params_dict: parameters to override the defaults in the config files | |
629 | :type params_dict: {str, str} | |
630 | :param regex_dict: regular expressions to use for matching the overriden parameters | |
631 | :type regex_dict: {str, str} or None | |
632 | :param str kind: "json" or "cnf" | |
633 | :param int timeout: arnied run verification timeout | |
634 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 635 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
636 | |
637 | The config files must be provided and are always expected to be found on | |
638 | the host. If these are absolute paths, they will be kept as is or | |
639 | otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided, | |
640 | the config files will be copied there as temporary files before applying. | |
641 | """ | |
642 | log.info("Performing semi-dynamic arnied configuration") | |
643 | ||
644 | config_paths = prep_cnf(config_files, params_dict, regex_dict) | |
645 | set_cnf(config_paths, kind=kind, timeout=timeout, vm=vm) | |
646 | ||
647 | log.info("Semi-dynamic arnied configuration successful!") | |
648 | ||
649 | ||
650 | def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None): | |
651 | """ | |
652 | Perform dynamic arnied configuration from fully generated config files. | |
653 | ||
654 | :param cnf: one key with the same value as *kind* and a list of cnfvars as value | |
655 | :type cnf: {str, str} | |
656 | :param config_file: optional user supplied filename | |
657 | :type config_file: str or None | |
658 | :param str kind: "json", "cnf", or "raw" | |
659 | :param int timeout: arnied run verification timeout | |
660 | :param vm: vm to run on if running on a guest instead of the host | |
f4c8f40e | 661 | :type vm: :py:class:`virttest.qemu_vm.VM` or None |
f49f6323 PD |
662 | :raises: :py:class:`ValueError` if `kind` is not an acceptable value |
663 | :raises: :py:class:`ConfigError` if cannot apply file | |
664 | ||
665 | The config file might not be provided in which case a temporary file will | |
666 | be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as | |
667 | an absolute path. If a vm is provided, the config file will be copied there | |
668 | as a temporary file before applying. | |
669 | """ | |
670 | if config_file is None: | |
671 | config_path = generate_config_path(dumped=True) | |
672 | elif os.path.isabs(config_file): | |
673 | config_path = config_file | |
674 | else: | |
675 | config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file) | |
676 | generated = config_file is None | |
677 | config_file = os.path.basename(config_path) | |
678 | log.info("Using %s cnf file %s%s", | |
679 | "generated" if generated else "user-supplied", | |
680 | config_file, " on %s" % vm.name if vm is not None else "") | |
681 | ||
02fe6179 | 682 | # Important to write bytes here to ensure text is encoded with latin-1 |
1626d17e | 683 | fd = open(config_path, "wb") |
f49f6323 PD |
684 | try: |
685 | SET_CNF_METHODS = { | |
7abff5a7 SA |
686 | "raw": cnfvar_old.write_cnf_raw, |
687 | "json": cnfvar_old.write_cnf_json, | |
688 | "cnf": cnfvar_old.write_cnf | |
f49f6323 PD |
689 | } |
690 | SET_CNF_METHODS[kind](cnf, out=fd) | |
691 | except KeyError: | |
692 | raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\"" | |
693 | % kind) | |
694 | finally: | |
695 | fd.close() | |
696 | log.info("Generated config file %s", config_path) | |
697 | ||
698 | kind = "cnf" if kind != "json" else kind | |
699 | set_cnf([config_path], kind=kind, timeout=timeout, vm=vm) | |
700 | ||
701 | ||
702 | def set_cnf_pipe(cnf, timeout=30, block=False): | |
703 | """ | |
704 | Set local configuration by talking to arnied via ``set_cnf``. | |
705 | ||
706 | :param cnf: one key with the same value as *kind* and a list of cnfvars as value | |
707 | :type cnf: {str, str} | |
708 | :param int timeout: arnied run verification timeout | |
709 | :param bool block: whether to wait for generate to complete the | |
710 | configuration change | |
711 | :returns: whether ``set_cnf`` succeeded or not | |
712 | :rtype: bool | |
713 | ||
714 | This is obviously not generic but supposed to be run on the guest. | |
715 | """ | |
716 | log.info("Setting arnied configuration through local pipe") | |
f1ca3964 | 717 | wait_for_arnied(timeout=timeout) |
f49f6323 PD |
718 | |
719 | st, out, exit = sysmisc.run_cmd_with_pipe([BIN_SET_CNF, "-j"], inp=str(cnf)) | |
720 | ||
721 | if st is False: | |
722 | log.error("Error applying configuration; status=%r" % exit) | |
723 | log.error("and stderr:\n%s" % out) | |
724 | return False | |
725 | log.debug("Configuration successfully passed to set_cnf, " | |
726 | "read %d B from pipe" % len(out)) | |
727 | ||
728 | if block is True: | |
729 | log.debug("Waiting for config job to complete") | |
730 | wait_for_generate() | |
731 | ||
732 | log.debug("Exiting sucessfully") | |
733 | return True | |
734 | ||
735 | ||
736 | def prep_config_paths(config_files, config_dir=None): | |
737 | """ | |
738 | Prepare absolute paths for all configs at an expected location. | |
739 | ||
740 | :param config_files: config files to use for the configuration | |
741 | :type config_files: [str] | |
742 | :param config_dir: config directory to prepend to the filepaths | |
743 | :type config_dir: str or None | |
744 | :returns: list of the full config paths | |
745 | :rtype: [str] | |
746 | """ | |
747 | if config_dir is None: | |
748 | config_dir = SRC_CONFIG_DIR | |
749 | config_paths = [] | |
750 | for config_file in config_files: | |
751 | if os.path.isabs(config_file): | |
752 | # Absolute path: The user requested a specific file | |
753 | # f.e. needed for dynamic arnied config update | |
754 | config_path = config_file | |
755 | else: | |
756 | config_path = os.path.join(os.path.abspath(config_dir), | |
757 | config_file) | |
758 | logging.debug("Using %s for original path %s", config_path, config_file) | |
759 | config_paths.append(config_path) | |
760 | return config_paths | |
761 | ||
762 | ||
763 | def prep_cnf_value(config_file, value, | |
764 | regex=None, template_key=None, ignore_fail=False): | |
765 | """ | |
766 | Replace value in a provided arnied config file. | |
767 | ||
768 | :param str config_file: file to use for the replacement | |
769 | :param str value: value to replace the first matched group with | |
770 | :param regex: regular expression to use when replacing a cnf value | |
771 | :type regex: str or None | |
772 | :param template_key: key of a quick template to use for the regex | |
773 | :type template_key: str or None | |
774 | :param bool ignore_fail: whether to ignore regex mismatching | |
775 | :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match | |
776 | ||
777 | In order to ensure better matching capabilities you are supposed to | |
778 | provide a regex pattern with at least one subgroup to match your value. | |
779 | What this means is that the value you like to replace is not directly | |
780 | searched into the config text but matched within a larger regex in | |
781 | in order to avoid any mismatch. | |
782 | ||
783 | Example: | |
784 | provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1 | |
785 | """ | |
786 | if template_key is None: | |
787 | pattern = regex.encode() | |
788 | else: | |
789 | samples = {"provider": 'PROVIDER_LOCALIP,\d+: "(\d+\.\d+\.\d+\.\d+)"', | |
790 | "global_destination_addr": 'SPAMFILTER_GLOBAL_DESTINATION_ADDR,0: "bounce_target@(.*)"'} | |
791 | pattern = samples[template_key].encode() | |
792 | ||
793 | with open(config_file, "rb") as file_handle: | |
794 | text = file_handle.read() | |
795 | match_line = re.search(pattern, text) | |
796 | ||
797 | if match_line is None and not ignore_fail: | |
798 | raise ValueError("Pattern %s not found in %s" % (pattern, config_file)) | |
799 | elif match_line is not None: | |
800 | old_line = match_line.group(0) | |
801 | text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):] | |
802 | line = re.search(pattern, text).group(0) | |
803 | log.debug("Updating %s to %s in %s", old_line, line, config_file) | |
804 | with open(config_file, "wb") as file_handle: | |
805 | file_handle.write(text) | |
806 | ||
807 | ||
808 | def prep_cnf(config_files, params_dict, regex_dict=None): | |
809 | """ | |
bcd9beb1 | 810 | Update all config files with the default overriding parameters, |
f49f6323 PD |
811 | i.e. override the values hard-coded in those config files. |
812 | ||
813 | :param config_files: config files to use for the configuration | |
814 | :type config_files: [str] | |
815 | :param params_dict: parameters to override the defaults in the config files | |
816 | :type params_dict: {str, str} | |
817 | :param regex_dict: regular expressions to use for matching the overriden parameters | |
818 | :type regex_dict: {str, str} or None | |
819 | :returns: list of prepared (modified) config paths | |
820 | :rtype: [str] | |
821 | """ | |
822 | log.info("Preparing %s template config files", len(config_files)) | |
823 | ||
824 | src_config_paths = prep_config_paths(config_files) | |
825 | new_config_paths = [] | |
826 | for config_path in src_config_paths: | |
827 | new_config_path = generate_config_path(dumped=True) | |
828 | shutil.copy(config_path, new_config_path) | |
829 | new_config_paths.append(new_config_path) | |
830 | ||
831 | for config_path in new_config_paths: | |
832 | for param_key in params_dict.keys(): | |
833 | if regex_dict is None: | |
834 | regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper() | |
835 | elif param_key in regex_dict.keys(): | |
836 | regex_val = regex_dict[param_key] % param_key.upper() | |
837 | elif re.match("\w*_\d+$", param_key): | |
838 | final_parameter, parent_id = \ | |
839 | re.match("(\w*)_(\d+)$", param_key).group(1, 2) | |
840 | regex_val = "\(%s\) %s,\d+: \"(.*)\"" \ | |
841 | % (parent_id, final_parameter.upper()) | |
842 | log.debug("Requested regex for %s is '%s'", | |
843 | param_key, regex_val) | |
844 | else: | |
845 | regex_val = "\s+%s,\d+: \"(.*)\"" % param_key.upper() | |
846 | prep_cnf_value(config_path, params_dict[param_key], | |
847 | regex=regex_val, ignore_fail=True) | |
848 | log.info("Prepared template config file %s", config_path) | |
849 | ||
850 | return new_config_paths | |
851 | ||
852 | ||
853 | def generate_config_path(dumped=False): | |
854 | """ | |
855 | Generate path for a temporary config name. | |
856 | ||
857 | :param bool dumped: whether the file should be in the dump | |
858 | directory or in temporary directory | |
859 | :returns: generated config file path | |
860 | :rtype: str | |
861 | """ | |
862 | dir = os.path.abspath(DUMP_CONFIG_DIR) if dumped else None | |
863 | fd, filename = tempfile.mkstemp(suffix=".cnf", dir=dir) | |
864 | os.close(fd) | |
865 | os.unlink(filename) | |
866 | return filename | |
867 | ||
868 | ||
f49f6323 PD |
869 | # enum |
870 | Delete = 0 | |
871 | Update = 1 | |
872 | Add = 2 | |
873 | Child = 3 | |
874 | ||
875 | ||
876 | def batch_update_cnf(cnf, vars): | |
877 | """ | |
878 | Perform a batch update of multiple cnf variables. | |
879 | ||
880 | :param cnf: CNF variable to update | |
881 | :type cnf: BuildCnfVar object | |
882 | :param vars: tuples of enumerated action and subtuple with data | |
883 | :type vars: [(int, (str, int, str))] | |
884 | :returns: updated CNF variable | |
885 | :rtype: BuildCnfVar object | |
886 | ||
887 | The actions are indexed in the same order: delete, update, add, child. | |
888 | """ | |
889 | last = 0 | |
890 | for (action, data) in vars: | |
891 | if action == Update: | |
892 | var, ref, val = data | |
893 | last = cnf.update_cnf(var, ref, val) | |
ff0ff458 | 894 | elif action == Add: |
f49f6323 PD |
895 | var, ref, val = data |
896 | last = cnf.add_cnf(var, ref, val) | |
897 | elif action == Delete: | |
898 | last = cnf.del_cnf(data) | |
899 | elif action == Child: # only one depth supported | |
900 | var, ref, val = data | |
901 | # do not update last | |
902 | cnf.add_cnf(var, ref, val, different_parent_line_no=last) | |
903 | return cnf | |
904 | ||
905 | ||
906 | def build_cnf(kind, instance=0, vals=[], data="", filename=None): | |
907 | """ | |
908 | Build a CNF variable and save it in a config file. | |
909 | ||
910 | :param str kind: name of the CNF variable | |
911 | :param int instance: instance number of the CNF variable | |
912 | :param vals: tuples of enumerated action and subtuple with data | |
913 | :type vals: [(int, (str, int, str))] | |
914 | :param str data: data for the CNF variable | |
915 | :param filename: optional custom name of the config file | |
916 | :type filename: str or None | |
917 | :returns: name of the saved config file | |
918 | :rtype: str | |
919 | """ | |
920 | builder = build_cnfvar.BuildCnfVar(kind, instance=instance, data=data) | |
921 | batch_update_cnf(builder, vals) | |
922 | filename = generate_config_path(dumped=True) if filename is None else filename | |
923 | [filename] = prep_config_paths([filename], DUMP_CONFIG_DIR) | |
924 | builder.save(filename) | |
925 | return filename | |
926 |