1 # This Python file uses the following encoding: utf-8
3 # The software in this package is distributed under the GNU General
4 # Public License version 2 (with a special exception described below).
6 # A copy of GNU General Public License (GPL) is included in this distribution,
7 # in the file COPYING.GPL.
9 # As a special exception, if other files instantiate templates or use macros
10 # or inline functions from this file, or you compile this file and link it
11 # with other works to produce a work based on this file, this file
12 # does not by itself cause the resulting work to be covered
13 # by the GNU General Public License.
15 # However the source code for this file must still be made available
16 # in accordance with section (3) of the GNU General Public License.
18 # This exception does not invalidate any other reasons why a work based
19 # on this file might be covered by the GNU General Public License.
21 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
26 ------------------------------------------------------
27 Miscellaneous system utility: Collection of various common system stuff / idioms.
29 Copyright: 2015 Intra2net AG
33 ------------------------------------------------------
34 The library exports the symbols below and some custom logging functions.
37 Wrapper for the default use case of the cumbersome "subprocess" library.
38 Accepts a list of arguments that describe the command invocation. Returns
39 ``True`` and the contents of ``stdout`` if the pipe returned sucessfully,
40 ``False`` plus ``stderr`` and the exit status otherwise. For example::
43 (success, output, _ret) = sysmisc.run_cmd_with_pipe([ "/usr/bin/date", "+%F" ])
45 print("Today is %s" % output)
47 print("Failed to read date from pipe.")
49 get_mountpoints_by_type
50 Extract mount points for the given file system type from */proc/mounts*.
51 Returns ``None`` if the file system is not mounted, a list of mount points
52 otherwise. Raises a test error if */proc/mounts* cannot be accessed.
55 Similar to run_cmd_with_pipe but allows processing of output line-by-line
56 as it becomes available. This may be necessary when the underlying binary
57 creates lots of output that cannot be buffered until the process finishes.
63 if re.match(r'\d', line):
64 print('found digits in line!')
65 sysmisc.read_linewise('dump_db', parse)
68 Return a hash of a file.
71 Replace the reboot binary with a fake one.
74 Run a command and wait until a condition evaluates to True.
77 A context manager that temporarily changes the current working directory
79 The logging functions either use the format capability or play
80 the simple role of providing shorter names.
84 ------------------------------------------------------
96 from contextlib import contextmanager
98 llog = logging.getLogger('pyi2ncommon.sysmisc')
101 __all__ = ("inf", "run_cmd_with_pipe", "get_mountpoints_by_type",
102 "read_linewise", "hash_file", "cheat_reboot",
103 "RUN_RESULT_OK", "RUN_RESULT_TIMEDOUT", "RUN_RESULT_FAIL",
104 "RUN_RESULT_NAME", "cmd_block_till")
107 ###############################################################################
109 ###############################################################################
114 A context manager which changes the working directory.
116 Changes current working directory to the given path, and then changes it
117 back to its previous value on exit.
119 Taken from comment for python recipe by Greg Warner at
120 http://code.activestate.com/recipes/576620-changedirectory-context-manager/
123 :arg str path: path to temporarily switch to
125 orig_wd = os.getcwd()
133 def run_cmd_with_pipe(argv, inp=None):
135 Read from a process pipe.
137 :param argv: arguments to use for creating a process
139 :param inp: Text to be piped into the program’s standard input.
141 :returns: a processes' stdout along with a status info
142 :rtype: bool * str * (int option)
144 Executes a binary and reads its output from a pipe. Returns a triple
145 encoding the programm success, its output either from stdout or stderr,
146 as well the exit status if non-zero.
148 If your process creates a lot of output, consider using
149 :py:func:`read_linewise` instead.
151 llog.debug("About to execute \"" + " ".join(argv) + "\"")
153 if isinstance(inp, str):
154 stdin = subprocess.PIPE
155 p = subprocess.Popen(argv,
157 stdout=subprocess.PIPE,
158 stderr=subprocess.PIPE)
159 if p.stdin is not None:
160 p.stdin.write(inp.encode())
161 (stdout, stderr) = p.communicate()
162 exit_status = p.wait()
164 return False, stderr.decode(), exit_status
165 return True, stdout.decode(), None
168 procmounts = "/proc/mounts"
171 def get_mountpoints_by_type(fstype):
173 Determine where some filesystem is mounted by reading the list
174 of mountpoints from */proc/mounts*.
176 :param str fstype: filesystem type
177 :returns: any mountpoints found
178 :rtype: str list option or None
179 :raises: :py:class:`IOError` if failed to read the process mounts
181 llog.debug("Determine mountpoints of %s." % fstype)
184 with open(procmounts, "r") as m:
186 pat = re.compile(r"^\S+\s+(\S+)\s+" + fstype + r"\s+.*$")
188 for mp in map(lambda l: re.match(pat, l), lines)
191 raise IOError(f"Failed to read {procmounts}")
197 def read_linewise(cmd, func, **kwargs):
199 Run `cmd` using subprocess, applying `func` to each line of stdout/stderr.
201 :param str cmd: command to read linewise
202 :param func: function to apply on each stdout line
204 :param kwargs: extra arguments for the subprocess initiation
205 :returns: the process' returncode from :py:meth:`subprocess.Popen.wait`
208 Creates a subprocess.Popen object with given `cmd`, `bufsize`=1, `stdout`=PIPE,
209 `stderr`=STDOUT, `universal_newlines`=True and the given `kwargs`.
211 As opposed to :py:func:`run_cmd_with_pipe`, output is not gathered and
212 returned but processed as it becomes available and then discared. This
213 allows to process output even if there is much of it.
215 proc = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE,
216 stderr=subprocess.STDOUT, universal_newlines=True,
219 for line in proc.stdout:
221 # (raises a ValueError in communicate() line:
222 # "Mixing iteration and read methods would lose data")
223 # if proc.poll() is not None:
226 # rest_output,_ = proc.communicate()
227 # for line in rest_output:
233 def hash_file(fname, new=hashlib.sha512, bs=4096):
237 :param str fname: name of the file to hash
238 :param new: constructor algorithm
239 :type new: builtin_function_or_method
240 :param int bs: read and write up to 'bs' bytes at a time
241 :returns: hexadecimal digest of the strings read from the file
245 with open(fname, "rb") as fp:
250 return hsh.hexdigest()
257 path="%s" # <- location of original executable, == location of script
258 backup="%s" # <- location of backup
259 chksum="%s" # <- sha512(orig_reboot)
263 echo "[$(date '+%%F %%T')] $*" &>> "${log}"
266 msg "Fake reboot invoked. Restoring executable from ${backup}."
268 if [ "${chksum}" = "$(sha512sum ${path} | cut -d ' ' -f 1)" ]; then
269 msg "Real reboot executable already in place at ${path}. Aborting."
273 if [ ! -x "${backup}" ]; then
274 msg "No backup executable at ${backup}!."
278 if [ "${chksum}" = "$(sha512sum ${backup} | cut -d ' ' -f 1)" ]
280 msg "Installing backup executable from ${backup} as ${path}."
281 if ! mv -f "${backup}" "${path}"; then
282 msg "Failed to replace ${path}."
285 msg "Checksum mismatch of ${backup}: Expected ${chksum}."
289 msg "Fake reboot successful -- next invocation will reboot indeed."
293 backup_infix = "backup"
294 backup_fmt = "%s.%s_%s"
295 logfile = "/var/log/cheat_reboot.log"
302 :raises: :py:class:`exceptions.OSError` if backup target already exists
304 This replaces the ``reboot-intranator`` executable by script which
305 replaces itself by the backed up executable upon the next invocation.
307 # path = utils.system_output("which reboot")
308 path = "/usr/intranator/bin/reboot-intranator"
309 suffix = uuid.uuid1()
310 backup = backup_fmt % (path, backup_infix, suffix)
312 if os.path.exists(backup):
313 raise OSError("Target %s already exists." % backup)
315 hexa = hash_file(path)
316 llog.debug("Found reboot at %s, hash %s; dst %s." % (path, hexa, backup))
317 subprocess.check_call(["mv", "-f", path, backup]) # backup existing binary
318 script = cheat_tpl % (path, backup, hexa, logfile)
319 with open(path, "w") as fp:
320 fp.write(script) # write script content to original location
321 mode = os.stat(path).st_mode
322 os.chmod(path, mode | stat.S_IXUSR | stat.S_IXGRP) # ug+x
325 RUN_RESULT_OK = 0 # → success, cond returned True within timeout
326 RUN_RESULT_TIMEDOUT = 1 # → success, but timeout elapsed
327 RUN_RESULT_FAIL = 2 # → fail
330 {RUN_RESULT_OK: "RUN_RESULT_OK", RUN_RESULT_TIMEDOUT: "RUN_RESULT_TIMEDOUT", RUN_RESULT_FAIL: "RUN_RESULT_FAIL"}
334 def cmd_block_till(cmd, timeout, cond, interval=1, *userdata, **kwuserdata):
336 Run ``cmd`` and wait until :py:func`cond` evaluates to True.
338 :param cmd: Command line or callback to execute. Function arguments must
339 have the same signature as :py:func:`run_cmd_with_pipe`.
340 :type cmd: [str] | types.FunctionType
341 :param int timeout: Blocking timeout
342 :param cond: Function to call; code will wait for this to return something
344 :param interval: Time (in seconds) to sleep between each attempt at `cond`
345 :returns: A Pair of result and error message if appropriate or `None`.
346 :rtype: (run_result, str | None)
348 llog.debug("cmd_block_till: %r, %d s, %r", cmd, timeout, cond)
349 if isinstance(cmd, types.FunctionType):
351 elif isinstance(cmd, list):
352 succ, out, _ = run_cmd_with_pipe(cmd) # caution: never pass further arguments!
354 raise TypeError("cmd_block_till: invalid type (cmd=%r); expected "
355 "function or argv", cmd)
358 return RUN_RESULT_FAIL, "cmd_block_till: invalid timeout; nonnegative " \
361 return RUN_RESULT_FAIL, "cmd_block_till: command %r failed (%s)" \
363 t_0 = time.time() # brr; cf. PEP 418 as to why
364 while cond(*userdata, **kwuserdata) is False:
368 return RUN_RESULT_TIMEDOUT, "cmd_block_till: command %r exceeded " \
369 "%d s timeout" % (cmd, timeout)
370 llog.debug("cmd_block_till: condition not satisfied after %d s, "
371 "retrying for another %d s" % (dt, timeout - dt))
373 return RUN_RESULT_OK, None
376 def replace_file_regex(edited_file, value, regex=None, ignore_fail=False):
378 Replace with value in a provided file using an optional regex or entirely.
380 :param str edited_file: file to use for the replacement
381 :param str value: value to replace the first matched group with
382 :param regex: more restrictive regular expression to use when replacing with value
383 :type regex: str or None
384 :param bool ignore_fail: whether to ignore regex mismatching
385 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
387 In order to ensure better matching capabilities you are supposed to
388 provide a regex pattern with at least one subgroup to match your value.
389 What this means is that the value you like to replace is not directly
390 searched into the config text but matched within a larger regex in
391 in order to avoid any mismatch.
394 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
396 pattern = regex.encode() if regex else "(.+)"
398 with open(edited_file, "rb") as file_handle:
399 text = file_handle.read()
400 match_line = re.search(pattern, text)
402 if match_line is None and not ignore_fail:
403 raise ValueError(f"Pattern {pattern} not found in {edited_file}")
404 elif match_line is not None:
405 old_line = match_line.group(0)
406 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
407 line = re.search(pattern, text).group(0)
408 llog.debug(f"Updating {old_line} to {line} in {edited_file}")
409 with open(edited_file, "wb") as file_handle:
410 file_handle.write(text)
413 ###############################################################################
415 ###############################################################################
417 CURRENT_TEST_STAGE = None
418 CURRENT_TEST_NAME = None
419 LOG_TAG = "%s/%s" % (os.path.basename(__file__), os.uname()[1])
423 def enter_test_stage(s):
424 """Group events into stages for status updates."""
425 global CURRENT_TEST_STAGE
426 CURRENT_TEST_STAGE = s
427 llog.info("Transitioning to test stage %s", s)
430 def progress(fmt, *args):
431 """Status updates that stand out among the log noise."""
432 if isinstance(CURRENT_TEST_STAGE, str):
433 label = "/%s" % CURRENT_TEST_STAGE
436 name = CURRENT_TEST_NAME if isinstance(CURRENT_TEST_NAME, str) else ""
437 fmt, label = str(fmt), str(label) # yes
438 llog.info("[%s%s] %s" % (name, label, fmt), *args)
439 # TODO: this method is more dynamic
440 # llog.info("[%s%s] %s%s" % (LOG_TAG, "", LOG_INDENT*indent, fmt), *args)
443 # these methods serve as shorter names
445 """Short name for INFO logging."""
446 llog.info(fmt, *args)
450 """Short name for DEBUG logging."""
451 llog.debug(fmt, *args)
455 """Short name for ERROR logging."""
456 llog.error(fmt, *args)
460 """Short name for WARN logging."""
461 llog.error(fmt, *args)
464 # these methods use the format capability
465 def log(level, text, *args, **kwargs):
466 """Log at any level using format capability."""
467 llog.log(level, text.format(*args), **kwargs)
470 def info(text, *args, **kwargs):
471 """Log at INFO level using format capability."""
472 log(logging.INFO, text, *args, **kwargs)
475 def debug(text, *args, **kwargs):
476 """Log at DEBUG level using format capability."""
477 log(logging.DEBUG, text, *args, **kwargs)
480 def error(text, *args, **kwargs):
481 """Log at ERROR level using format capability."""
482 log(logging.ERROR, text, *args, **kwargs)
485 def warn(text, *args, **kwargs):
486 """Log at WARN level using format capability."""
487 log(logging.WARN, text, *args, **kwargs)