Commit | Line | Data |
---|---|---|
f49f6323 PD |
1 | # This Python file uses the following encoding: utf-8 |
2 | ||
11cbb815 PD |
3 | # The software in this package is distributed under the GNU General |
4 | # Public License version 2 (with a special exception described below). | |
5 | # | |
6 | # A copy of GNU General Public License (GPL) is included in this distribution, | |
7 | # in the file COPYING.GPL. | |
8 | # | |
9 | # As a special exception, if other files instantiate templates or use macros | |
10 | # or inline functions from this file, or you compile this file and link it | |
11 | # with other works to produce a work based on this file, this file | |
12 | # does not by itself cause the resulting work to be covered | |
13 | # by the GNU General Public License. | |
14 | # | |
15 | # However the source code for this file must still be made available | |
16 | # in accordance with section (3) of the GNU General Public License. | |
17 | # | |
18 | # This exception does not invalidate any other reasons why a work based | |
19 | # on this file might be covered by the GNU General Public License. | |
20 | # | |
21 | # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com> | |
22 | ||
f49f6323 PD |
23 | """ |
24 | ||
25 | SUMMARY | |
26 | ------------------------------------------------------ | |
27 | Miscellaneous system utility: Collection of various common system stuff / idioms. | |
28 | ||
29 | Copyright: 2015 Intra2net AG | |
30 | ||
31 | ||
32 | CONTENTS | |
33 | ------------------------------------------------------ | |
34 | The library exports the symbols below and some custom logging functions. | |
35 | ||
36 | run_cmd_with_pipe | |
37 | Wrapper for the default use case of the cumbersome "subprocess" library. | |
7628bc48 | 38 | Accepts a list of arguments that describe the command invocation. Returns |
f49f6323 PD |
39 | ``True`` and the contents of ``stdout`` if the pipe returned sucessfully, |
40 | ``False`` plus ``stderr`` and the exit status otherwise. For example:: | |
41 | ||
42 | import sysmisc | |
43 | (success, output, _ret) = sysmisc.run_cmd_with_pipe([ "/usr/bin/date", "+%F" ]) | |
44 | if success is True: | |
45 | print("Today is %s" % output) | |
46 | else: | |
47 | print("Failed to read date from pipe.") | |
48 | ||
49 | get_mountpoints_by_type | |
50 | Extract mount points for the given file system type from */proc/mounts*. | |
51 | Returns ``None`` if the file system is not mounted, a list of mount points | |
52 | otherwise. Raises a test error if */proc/mounts* cannot be accessed. | |
53 | ||
54 | read_linewise | |
55 | Similar to run_cmd_with_pipe but allows processing of output line-by-line | |
56 | as it becomes available. This may be necessary when the underlying binary | |
57 | creates lots of output that cannot be buffered until the process finishes. | |
58 | Example:: | |
59 | ||
60 | import re | |
61 | import sysmisc | |
62 | def parse(line): | |
7628bc48 | 63 | if re.match(r'\d', line): |
f49f6323 PD |
64 | print('found digits in line!') |
65 | sysmisc.read_linewise('dump_db', parse) | |
66 | ||
67 | hash_file | |
7628bc48 | 68 | Return a hash of a file. |
f49f6323 PD |
69 | |
70 | cheat_reboot | |
71 | Replace the reboot binary with a fake one. | |
72 | ||
73 | cmd_block_till | |
74 | Run a command and wait until a condition evaluates to True. | |
75 | ||
872eaff1 CH |
76 | cd |
77 | A context manager that temporarily changes the current working directory | |
78 | ||
f49f6323 PD |
79 | The logging functions either use the format capability or play |
80 | the simple role of providing shorter names. | |
81 | ||
82 | ||
83 | INTERFACE | |
84 | ------------------------------------------------------ | |
85 | ||
86 | """ | |
87 | ||
f49f6323 PD |
88 | import re |
89 | import subprocess | |
90 | import hashlib | |
91 | import os | |
92 | import stat | |
93 | import time | |
94 | import types | |
95 | import uuid | |
872eaff1 | 96 | from contextlib import contextmanager |
f49f6323 | 97 | import logging |
3de8b4d8 | 98 | llog = logging.getLogger('pyi2ncommon.sysmisc') |
f49f6323 PD |
99 | |
100 | ||
7628bc48 CH |
101 | __all__ = ("inf", "run_cmd_with_pipe", "get_mountpoints_by_type", |
102 | "read_linewise", "hash_file", "cheat_reboot", | |
103 | "RUN_RESULT_OK", "RUN_RESULT_TIMEDOUT", "RUN_RESULT_FAIL", | |
104 | "RUN_RESULT_NAME", "cmd_block_till") | |
f49f6323 PD |
105 | |
106 | ||
107 | ############################################################################### | |
108 | # HELPERS | |
109 | ############################################################################### | |
110 | ||
872eaff1 CH |
111 | @contextmanager |
112 | def cd(path): | |
113 | """ | |
114 | A context manager which changes the working directory. | |
115 | ||
116 | Changes current working directory to the given path, and then changes it | |
117 | back to its previous value on exit. | |
118 | ||
119 | Taken from comment for python recipe by Greg Warner at | |
120 | http://code.activestate.com/recipes/576620-changedirectory-context-manager/ | |
121 | (MIT license) | |
79da5258 | 122 | |
7628bc48 | 123 | :arg str path: path to temporarily switch to |
872eaff1 CH |
124 | """ |
125 | orig_wd = os.getcwd() | |
126 | os.chdir(path) | |
127 | try: | |
128 | yield | |
129 | finally: | |
130 | os.chdir(orig_wd) | |
131 | ||
132 | ||
f49f6323 PD |
133 | def run_cmd_with_pipe(argv, inp=None): |
134 | """ | |
135 | Read from a process pipe. | |
136 | ||
137 | :param argv: arguments to use for creating a process | |
138 | :type argv: [str] | |
139 | :param inp: Text to be piped into the program’s standard input. | |
140 | :type inp: str | |
141 | :returns: a processes' stdout along with a status info | |
142 | :rtype: bool * str * (int option) | |
143 | ||
144 | Executes a binary and reads its output from a pipe. Returns a triple | |
145 | encoding the programm success, its output either from stdout or stderr, | |
146 | as well the exit status if non-zero. | |
147 | ||
148 | If your process creates a lot of output, consider using | |
149 | :py:func:`read_linewise` instead. | |
150 | """ | |
151 | llog.debug("About to execute \"" + " ".join(argv) + "\"") | |
152 | stdin = None | |
153 | if isinstance(inp, str): | |
154 | stdin = subprocess.PIPE | |
155 | p = subprocess.Popen(argv, | |
156 | stdin=stdin, | |
157 | stdout=subprocess.PIPE, | |
158 | stderr=subprocess.PIPE) | |
159 | if p.stdin is not None: | |
160 | p.stdin.write(inp.encode()) | |
161 | (stdout, stderr) = p.communicate() | |
162 | exit_status = p.wait() | |
163 | if exit_status != 0: | |
164 | return False, stderr.decode(), exit_status | |
165 | return True, stdout.decode(), None | |
166 | ||
167 | ||
168 | procmounts = "/proc/mounts" | |
169 | ||
170 | ||
171 | def get_mountpoints_by_type(fstype): | |
172 | """ | |
173 | Determine where some filesystem is mounted by reading the list | |
174 | of mountpoints from */proc/mounts*. | |
175 | ||
176 | :param str fstype: filesystem type | |
177 | :returns: any mountpoints found | |
178 | :rtype: str list option or None | |
179 | :raises: :py:class:`IOError` if failed to read the process mounts | |
180 | """ | |
181 | llog.debug("Determine mountpoints of %s." % fstype) | |
182 | mps = None | |
183 | try: | |
184 | with open(procmounts, "r") as m: | |
185 | lines = list(m) | |
7628bc48 | 186 | pat = re.compile(r"^\S+\s+(\S+)\s+" + fstype + r"\s+.*$") |
f49f6323 PD |
187 | mps = [mp.group(1) |
188 | for mp in map(lambda l: re.match(pat, l), lines) | |
189 | if mp] | |
7628bc48 CH |
190 | except IOError: |
191 | raise IOError(f"Failed to read {procmounts}") | |
f49f6323 PD |
192 | if not mps: |
193 | return None | |
194 | return mps | |
195 | ||
196 | ||
197 | def read_linewise(cmd, func, **kwargs): | |
198 | """ | |
199 | Run `cmd` using subprocess, applying `func` to each line of stdout/stderr. | |
200 | ||
201 | :param str cmd: command to read linewise | |
202 | :param func: function to apply on each stdout line | |
203 | :type func: function | |
204 | :param kwargs: extra arguments for the subprocess initiation | |
205 | :returns: the process' returncode from :py:meth:`subprocess.Popen.wait` | |
206 | :rtype: int | |
207 | ||
208 | Creates a subprocess.Popen object with given `cmd`, `bufsize`=1, `stdout`=PIPE, | |
209 | `stderr`=STDOUT, `universal_newlines`=True and the given `kwargs`. | |
210 | ||
211 | As opposed to :py:func:`run_cmd_with_pipe`, output is not gathered and | |
212 | returned but processed as it becomes available and then discared. This | |
213 | allows to process output even if there is much of it. | |
214 | """ | |
215 | proc = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE, | |
216 | stderr=subprocess.STDOUT, universal_newlines=True, | |
217 | **kwargs) | |
218 | ||
219 | for line in proc.stdout: | |
220 | func(line) | |
221 | # (raises a ValueError in communicate() line: | |
222 | # "Mixing iteration and read methods would lose data") | |
223 | # if proc.poll() is not None: | |
224 | # break | |
225 | ||
7628bc48 CH |
226 | # rest_output,_ = proc.communicate() |
227 | # for line in rest_output: | |
228 | # func(line) | |
f49f6323 PD |
229 | |
230 | return proc.wait() | |
231 | ||
232 | ||
233 | def hash_file(fname, new=hashlib.sha512, bs=4096): | |
234 | """ | |
235 | Return a file hash. | |
236 | ||
237 | :param str fname: name of the file to hash | |
238 | :param new: constructor algorithm | |
239 | :type new: builtin_function_or_method | |
240 | :param int bs: read and write up to 'bs' bytes at a time | |
241 | :returns: hexadecimal digest of the strings read from the file | |
242 | :rtype: str | |
243 | """ | |
244 | hsh = new() | |
245 | with open(fname, "rb") as fp: | |
246 | buf = fp.read(bs) | |
247 | while len(buf) > 0: | |
248 | hsh.update(buf) | |
249 | buf = fp.read(bs) | |
250 | return hsh.hexdigest() | |
251 | ||
252 | ||
253 | cheat_tpl = """\ | |
254 | #!/bin/sh | |
255 | set -u | |
256 | ||
257 | path="%s" # <- location of original executable, == location of script | |
258 | backup="%s" # <- location of backup | |
259 | chksum="%s" # <- sha512(orig_reboot) | |
260 | log="%s" # <- stdout | |
261 | ||
262 | msg () { | |
263 | echo "[$(date '+%%F %%T')] $*" &>> "${log}" | |
264 | } | |
265 | ||
266 | msg "Fake reboot invoked. Restoring executable from ${backup}." | |
267 | ||
268 | if [ "${chksum}" = "$(sha512sum ${path} | cut -d ' ' -f 1)" ]; then | |
269 | msg "Real reboot executable already in place at ${path}. Aborting." | |
270 | exit 1 | |
271 | fi | |
272 | ||
273 | if [ ! -x "${backup}" ]; then | |
274 | msg "No backup executable at ${backup}!." | |
275 | exit 1 | |
276 | fi | |
277 | ||
278 | if [ "${chksum}" = "$(sha512sum ${backup} | cut -d ' ' -f 1)" ] | |
279 | then | |
280 | msg "Installing backup executable from ${backup} as ${path}." | |
281 | if ! mv -f "${backup}" "${path}"; then | |
282 | msg "Failed to replace ${path}." | |
283 | fi | |
284 | else | |
285 | msg "Checksum mismatch of ${backup}: Expected ${chksum}." | |
286 | exit 1 | |
287 | fi | |
288 | ||
289 | msg "Fake reboot successful -- next invocation will reboot indeed." | |
290 | ||
291 | """ | |
292 | ||
293 | backup_infix = "backup" | |
294 | backup_fmt = "%s.%s_%s" | |
295 | logfile = "/var/log/cheat_reboot.log" | |
296 | ||
297 | ||
298 | def cheat_reboot(): | |
299 | """ | |
300 | Skip one reboot. | |
301 | ||
302 | :raises: :py:class:`exceptions.OSError` if backup target already exists | |
303 | ||
bdb845b9 PD |
304 | This replaces the ``reboot-intranator`` executable by script which |
305 | replaces itself by the backed up executable upon the next invocation. | |
f49f6323 | 306 | """ |
7628bc48 | 307 | # path = utils.system_output("which reboot") |
f49f6323 PD |
308 | path = "/usr/intranator/bin/reboot-intranator" |
309 | suffix = uuid.uuid1() | |
310 | backup = backup_fmt % (path, backup_infix, suffix) | |
311 | ||
312 | if os.path.exists(backup): | |
313 | raise OSError("Target %s already exists." % backup) | |
314 | ||
315 | hexa = hash_file(path) | |
316 | llog.debug("Found reboot at %s, hash %s; dst %s." % (path, hexa, backup)) | |
317 | subprocess.check_call(["mv", "-f", path, backup]) # backup existing binary | |
318 | script = cheat_tpl % (path, backup, hexa, logfile) | |
319 | with open(path, "w") as fp: | |
320 | fp.write(script) # write script content to original location | |
321 | mode = os.stat(path).st_mode | |
322 | os.chmod(path, mode | stat.S_IXUSR | stat.S_IXGRP) # ug+x | |
323 | ||
324 | ||
325 | RUN_RESULT_OK = 0 # → success, cond returned True within timeout | |
326 | RUN_RESULT_TIMEDOUT = 1 # → success, but timeout elapsed | |
327 | RUN_RESULT_FAIL = 2 # → fail | |
328 | ||
329 | RUN_RESULT_NAME = ( | |
330 | {RUN_RESULT_OK: "RUN_RESULT_OK", RUN_RESULT_TIMEDOUT: "RUN_RESULT_TIMEDOUT", RUN_RESULT_FAIL: "RUN_RESULT_FAIL"} | |
331 | ) | |
332 | ||
333 | ||
334 | def cmd_block_till(cmd, timeout, cond, interval=1, *userdata, **kwuserdata): | |
335 | """ | |
336 | Run ``cmd`` and wait until :py:func`cond` evaluates to True. | |
337 | ||
338 | :param cmd: Command line or callback to execute. Function arguments must | |
339 | have the same signature as :py:func:`run_cmd_with_pipe`. | |
7628bc48 | 340 | :type cmd: [str] | types.FunctionType |
f49f6323 | 341 | :param int timeout: Blocking timeout |
7628bc48 CH |
342 | :param cond: Function to call; code will wait for this to return something |
343 | other than `False` | |
344 | :param interval: Time (in seconds) to sleep between each attempt at `cond` | |
df036fbe | 345 | :returns: A Pair of result and error message if appropriate or `None`. |
7628bc48 | 346 | :rtype: (run_result, str | None) |
f49f6323 PD |
347 | """ |
348 | llog.debug("cmd_block_till: %r, %d s, %r", cmd, timeout, cond) | |
349 | if isinstance(cmd, types.FunctionType): | |
350 | succ, out, _ = cmd() | |
351 | elif isinstance(cmd, list): | |
352 | succ, out, _ = run_cmd_with_pipe(cmd) # caution: never pass further arguments! | |
353 | else: | |
354 | raise TypeError("cmd_block_till: invalid type (cmd=%r); expected " | |
355 | "function or argv", cmd) | |
356 | ||
357 | if timeout < 0: | |
358 | return RUN_RESULT_FAIL, "cmd_block_till: invalid timeout; nonnegative " \ | |
359 | "integer expected" | |
360 | if succ is False: | |
361 | return RUN_RESULT_FAIL, "cmd_block_till: command %r failed (%s)" \ | |
362 | % (cmd, str(out)) | |
363 | t_0 = time.time() # brr; cf. PEP 418 as to why | |
364 | while cond(*userdata, **kwuserdata) is False: | |
365 | t_now = time.time() | |
366 | dt = t_now - t_0 | |
367 | if dt > timeout: | |
368 | return RUN_RESULT_TIMEDOUT, "cmd_block_till: command %r exceeded " \ | |
369 | "%d s timeout" % (cmd, timeout) | |
370 | llog.debug("cmd_block_till: condition not satisfied after %d s, " | |
371 | "retrying for another %d s" % (dt, timeout - dt)) | |
372 | time.sleep(interval) | |
373 | return RUN_RESULT_OK, None | |
374 | ||
375 | ||
4965c436 PD |
376 | def replace_file_regex(edited_file, value, regex=None, ignore_fail=False): |
377 | """ | |
378 | Replace with value in a provided file using an optional regex or entirely. | |
379 | ||
380 | :param str edited_file: file to use for the replacement | |
381 | :param str value: value to replace the first matched group with | |
382 | :param regex: more restrictive regular expression to use when replacing with value | |
383 | :type regex: str or None | |
384 | :param bool ignore_fail: whether to ignore regex mismatching | |
385 | :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match | |
386 | ||
387 | In order to ensure better matching capabilities you are supposed to | |
388 | provide a regex pattern with at least one subgroup to match your value. | |
389 | What this means is that the value you like to replace is not directly | |
390 | searched into the config text but matched within a larger regex in | |
391 | in order to avoid any mismatch. | |
392 | ||
393 | Example: | |
394 | provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1 | |
395 | """ | |
396 | pattern = regex.encode() if regex else "(.+)" | |
397 | ||
398 | with open(edited_file, "rb") as file_handle: | |
399 | text = file_handle.read() | |
400 | match_line = re.search(pattern, text) | |
401 | ||
402 | if match_line is None and not ignore_fail: | |
403 | raise ValueError(f"Pattern {pattern} not found in {edited_file}") | |
404 | elif match_line is not None: | |
405 | old_line = match_line.group(0) | |
406 | text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):] | |
407 | line = re.search(pattern, text).group(0) | |
408 | llog.debug(f"Updating {old_line} to {line} in {edited_file}") | |
409 | with open(edited_file, "wb") as file_handle: | |
410 | file_handle.write(text) | |
411 | ||
412 | ||
f49f6323 PD |
413 | ############################################################################### |
414 | # LOGGING | |
415 | ############################################################################### | |
416 | ||
417 | CURRENT_TEST_STAGE = None | |
418 | CURRENT_TEST_NAME = None | |
419 | LOG_TAG = "%s/%s" % (os.path.basename(__file__), os.uname()[1]) | |
420 | LOG_INDENT = " " | |
421 | ||
422 | ||
423 | def enter_test_stage(s): | |
424 | """Group events into stages for status updates.""" | |
425 | global CURRENT_TEST_STAGE | |
426 | CURRENT_TEST_STAGE = s | |
427 | llog.info("Transitioning to test stage %s", s) | |
428 | ||
429 | ||
430 | def progress(fmt, *args): | |
431 | """Status updates that stand out among the log noise.""" | |
432 | if isinstance(CURRENT_TEST_STAGE, str): | |
433 | label = "/%s" % CURRENT_TEST_STAGE | |
434 | else: | |
435 | label = "" | |
436 | name = CURRENT_TEST_NAME if isinstance(CURRENT_TEST_NAME, str) else "" | |
7628bc48 | 437 | fmt, label = str(fmt), str(label) # yes |
f49f6323 PD |
438 | llog.info("[%s%s] %s" % (name, label, fmt), *args) |
439 | # TODO: this method is more dynamic | |
440 | # llog.info("[%s%s] %s%s" % (LOG_TAG, "", LOG_INDENT*indent, fmt), *args) | |
441 | ||
442 | ||
443 | # these methods serve as shorter names | |
444 | def inf(fmt, *args): | |
445 | """Short name for INFO logging.""" | |
446 | llog.info(fmt, *args) | |
447 | ||
448 | ||
449 | def dbg(fmt, *args): | |
450 | """Short name for DEBUG logging.""" | |
451 | llog.debug(fmt, *args) | |
452 | ||
453 | ||
454 | def err(fmt, *args): | |
455 | """Short name for ERROR logging.""" | |
456 | llog.error(fmt, *args) | |
457 | ||
458 | ||
459 | def wrn(fmt, *args): | |
460 | """Short name for WARN logging.""" | |
461 | llog.error(fmt, *args) | |
462 | ||
463 | ||
464 | # these methods use the format capability | |
465 | def log(level, text, *args, **kwargs): | |
466 | """Log at any level using format capability.""" | |
467 | llog.log(level, text.format(*args), **kwargs) | |
468 | ||
469 | ||
470 | def info(text, *args, **kwargs): | |
471 | """Log at INFO level using format capability.""" | |
472 | log(logging.INFO, text, *args, **kwargs) | |
473 | ||
474 | ||
475 | def debug(text, *args, **kwargs): | |
476 | """Log at DEBUG level using format capability.""" | |
477 | log(logging.DEBUG, text, *args, **kwargs) | |
478 | ||
479 | ||
480 | def error(text, *args, **kwargs): | |
481 | """Log at ERROR level using format capability.""" | |
482 | log(logging.ERROR, text, *args, **kwargs) | |
483 | ||
484 | ||
485 | def warn(text, *args, **kwargs): | |
486 | """Log at WARN level using format capability.""" | |
487 | log(logging.WARN, text, *args, **kwargs) |