1 # This Python file uses the following encoding: utf-8
3 # The software in this package is distributed under the GNU General
4 # Public License version 2 (with a special exception described below).
6 # A copy of GNU General Public License (GPL) is included in this distribution,
7 # in the file COPYING.GPL.
9 # As a special exception, if other files instantiate templates or use macros
10 # or inline functions from this file, or you compile this file and link it
11 # with other works to produce a work based on this file, this file
12 # does not by itself cause the resulting work to be covered
13 # by the GNU General Public License.
15 # However the source code for this file must still be made available
16 # in accordance with section (3) of the GNU General Public License.
18 # This exception does not invalidate any other reasons why a work based
19 # on this file might be covered by the GNU General Public License.
21 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 Miscellaneous system utility: Collection of various common system stuff / idioms.
26 Copyright: 2015 Intra2net AG
28 The library exports the symbols below and some custom logging functions.
31 Wrapper for the default use case of the cumbersome "subprocess" library.
32 Accepts a list of arguments that describe the command invocation. Returns
33 ``True`` and the contents of ``stdout`` if the pipe returned sucessfully,
34 ``False`` plus ``stderr`` and the exit status otherwise. For example::
37 (success, output, _ret) = sysmisc.run_cmd_with_pipe([ "/usr/bin/date", "+%F" ])
39 print("Today is %s" % output)
41 print("Failed to read date from pipe.")
43 * get_mountpoints_by_type
44 Extract mount points for the given file system type from */proc/mounts*.
45 Returns ``None`` if the file system is not mounted, a list of mount points
46 otherwise. Raises a test error if */proc/mounts* cannot be accessed.
49 Similar to run_cmd_with_pipe but allows processing of output line-by-line
50 as it becomes available. This may be necessary when the underlying binary
51 creates lots of output that cannot be buffered until the process finishes.
57 if re.match(r'\d', line):
58 print('found digits in line!')
59 sysmisc.read_linewise('dump_db', parse)
62 Return a hash of a file.
65 Replace the reboot binary with a fake one.
68 Run a command and wait until a condition evaluates to True.
71 A context manager that temporarily changes the current working directory
73 The logging functions either use the format capability or play
74 the simple role of providing shorter names.
85 from contextlib import contextmanager
87 llog = logging.getLogger('pyi2ncommon.sysmisc')
90 __all__ = ("inf", "run_cmd_with_pipe", "get_mountpoints_by_type",
91 "read_linewise", "hash_file", "cheat_reboot",
92 "RUN_RESULT_OK", "RUN_RESULT_TIMEDOUT", "RUN_RESULT_FAIL",
93 "RUN_RESULT_NAME", "cmd_block_till")
96 ###############################################################################
98 ###############################################################################
103 A context manager which changes the working directory.
105 Changes current working directory to the given path, and then changes it
106 back to its previous value on exit.
108 Taken from comment for python recipe by Greg Warner at
109 http://code.activestate.com/recipes/576620-changedirectory-context-manager/
112 :arg str path: path to temporarily switch to
114 orig_wd = os.getcwd()
122 def run_cmd_with_pipe(argv, inp=None):
124 Read from a process pipe.
126 :param argv: arguments to use for creating a process
128 :param inp: Text to be piped into the program’s standard input.
130 :returns: a processes' stdout along with a status info
131 :rtype: bool * str * (int option)
133 Executes a binary and reads its output from a pipe. Returns a triple
134 encoding the programm success, its output either from stdout or stderr,
135 as well the exit status if non-zero.
137 If your process creates a lot of output, consider using
138 :py:func:`read_linewise` instead.
140 llog.debug("About to execute \"" + " ".join(argv) + "\"")
142 if isinstance(inp, str):
143 stdin = subprocess.PIPE
144 p = subprocess.Popen(argv,
146 stdout=subprocess.PIPE,
147 stderr=subprocess.PIPE)
148 if p.stdin is not None:
149 p.stdin.write(inp.encode())
150 (stdout, stderr) = p.communicate()
151 exit_status = p.wait()
153 return False, stderr.decode(), exit_status
154 return True, stdout.decode(), None
157 procmounts = "/proc/mounts"
160 def get_mountpoints_by_type(fstype):
162 Determine where some filesystem is mounted by reading the list
163 of mountpoints from */proc/mounts*.
165 :param str fstype: filesystem type
166 :returns: any mountpoints found
167 :rtype: str list option or None
168 :raises: :py:class:`IOError` if failed to read the process mounts
170 llog.debug("Determine mountpoints of %s." % fstype)
173 with open(procmounts, "r") as m:
175 pat = re.compile(r"^\S+\s+(\S+)\s+" + fstype + r"\s+.*$")
177 for mp in map(lambda l: re.match(pat, l), lines)
180 raise IOError(f"Failed to read {procmounts}")
186 def read_linewise(cmd, func, **kwargs):
188 Run `cmd` using subprocess, applying `func` to each line of stdout/stderr.
190 :param str cmd: command to read linewise
191 :param func: function to apply on each stdout line
193 :param kwargs: extra arguments for the subprocess initiation
194 :returns: the process' returncode from :py:meth:`subprocess.Popen.wait`
197 Creates a subprocess.Popen object with given `cmd`, `bufsize`=1, `stdout`=PIPE,
198 `stderr`=STDOUT, `universal_newlines`=True and the given `kwargs`.
200 As opposed to :py:func:`run_cmd_with_pipe`, output is not gathered and
201 returned but processed as it becomes available and then discared. This
202 allows to process output even if there is much of it.
204 proc = subprocess.Popen(cmd, bufsize=1, stdout=subprocess.PIPE,
205 stderr=subprocess.STDOUT, universal_newlines=True,
208 for line in proc.stdout:
210 # (raises a ValueError in communicate() line:
211 # "Mixing iteration and read methods would lose data")
212 # if proc.poll() is not None:
215 # rest_output,_ = proc.communicate()
216 # for line in rest_output:
222 def hash_file(fname, new=hashlib.sha512, bs=4096):
226 :param str fname: name of the file to hash
227 :param new: constructor algorithm
228 :type new: builtin_function_or_method
229 :param int bs: read and write up to 'bs' bytes at a time
230 :returns: hexadecimal digest of the strings read from the file
234 with open(fname, "rb") as fp:
239 return hsh.hexdigest()
246 path="%s" # <- location of original executable, == location of script
247 backup="%s" # <- location of backup
248 chksum="%s" # <- sha512(orig_reboot)
252 echo "[$(date '+%%F %%T')] $*" &>> "${log}"
255 msg "Fake reboot invoked. Restoring executable from ${backup}."
257 if [ "${chksum}" = "$(sha512sum ${path} | cut -d ' ' -f 1)" ]; then
258 msg "Real reboot executable already in place at ${path}. Aborting."
262 if [ ! -x "${backup}" ]; then
263 msg "No backup executable at ${backup}!."
267 if [ "${chksum}" = "$(sha512sum ${backup} | cut -d ' ' -f 1)" ]
269 msg "Installing backup executable from ${backup} as ${path}."
270 if ! mv -f "${backup}" "${path}"; then
271 msg "Failed to replace ${path}."
274 msg "Checksum mismatch of ${backup}: Expected ${chksum}."
278 msg "Fake reboot successful -- next invocation will reboot indeed."
282 backup_infix = "backup"
283 backup_fmt = "%s.%s_%s"
284 logfile = "/var/log/cheat_reboot.log"
291 :raises: :py:class:`exceptions.OSError` if backup target already exists
293 This replaces the ``reboot-intranator`` executable by script which
294 replaces itself by the backed up executable upon the next invocation.
296 # path = utils.system_output("which reboot")
297 path = "/usr/intranator/bin/reboot-intranator"
298 suffix = uuid.uuid1()
299 backup = backup_fmt % (path, backup_infix, suffix)
301 if os.path.exists(backup):
302 raise OSError("Target %s already exists." % backup)
304 hexa = hash_file(path)
305 llog.debug("Found reboot at %s, hash %s; dst %s." % (path, hexa, backup))
306 subprocess.check_call(["mv", "-f", path, backup]) # backup existing binary
307 script = cheat_tpl % (path, backup, hexa, logfile)
308 with open(path, "w") as fp:
309 fp.write(script) # write script content to original location
310 mode = os.stat(path).st_mode
311 os.chmod(path, mode | stat.S_IXUSR | stat.S_IXGRP) # ug+x
314 RUN_RESULT_OK = 0 # → success, cond returned True within timeout
315 RUN_RESULT_TIMEDOUT = 1 # → success, but timeout elapsed
316 RUN_RESULT_FAIL = 2 # → fail
319 {RUN_RESULT_OK: "RUN_RESULT_OK", RUN_RESULT_TIMEDOUT: "RUN_RESULT_TIMEDOUT", RUN_RESULT_FAIL: "RUN_RESULT_FAIL"}
323 def cmd_block_till(cmd, timeout, cond, interval=1, *userdata, **kwuserdata):
325 Run ``cmd`` and wait until :py:func`cond` evaluates to True.
327 :param cmd: Command line or callback to execute. Function arguments must
328 have the same signature as :py:func:`run_cmd_with_pipe`.
329 :type cmd: [str] | types.FunctionType
330 :param int timeout: Blocking timeout
331 :param cond: Function to call; code will wait for this to return something
333 :param interval: Time (in seconds) to sleep between each attempt at `cond`
334 :returns: A Pair of result and error message if appropriate or `None`.
335 :rtype: (run_result, str | None)
337 llog.debug("cmd_block_till: %r, %d s, %r", cmd, timeout, cond)
338 if isinstance(cmd, types.FunctionType):
340 elif isinstance(cmd, list):
341 succ, out, _ = run_cmd_with_pipe(cmd) # caution: never pass further arguments!
343 raise TypeError("cmd_block_till: invalid type (cmd=%r); expected "
344 "function or argv", cmd)
347 return RUN_RESULT_FAIL, "cmd_block_till: invalid timeout; nonnegative " \
350 return RUN_RESULT_FAIL, "cmd_block_till: command %r failed (%s)" \
352 t_0 = time.time() # brr; cf. PEP 418 as to why
353 while cond(*userdata, **kwuserdata) is False:
357 return RUN_RESULT_TIMEDOUT, "cmd_block_till: command %r exceeded " \
358 "%d s timeout" % (cmd, timeout)
359 llog.debug("cmd_block_till: condition not satisfied after %d s, "
360 "retrying for another %d s" % (dt, timeout - dt))
362 return RUN_RESULT_OK, None
365 def replace_file_regex(edited_file, value, regex=None, ignore_fail=False):
367 Replace with value in a provided file using an optional regex or entirely.
369 :param str edited_file: file to use for the replacement
370 :param str value: value to replace the first matched group with
371 :param regex: more restrictive regular expression to use when replacing with value
372 :type regex: str or None
373 :param bool ignore_fail: whether to ignore regex mismatching
374 :raises: :py:class:`ValueError` if (also default) `regex` doesn't have a match
376 In order to ensure better matching capabilities you are supposed to
377 provide a regex pattern with at least one subgroup to match your value.
378 What this means is that the value you like to replace is not directly
379 searched into the config text but matched within a larger regex in
380 in order to avoid any mismatch.
383 provider.cnf, 'PROVIDER_LOCALIP,0: "(\d+)"', 127.0.0.1
385 pattern = regex.encode() if regex else "(.+)"
387 with open(edited_file, "rb") as file_handle:
388 text = file_handle.read()
389 match_line = re.search(pattern, text)
391 if match_line is None and not ignore_fail:
392 raise ValueError(f"Pattern {pattern} not found in {edited_file}")
393 elif match_line is not None:
394 old_line = match_line.group(0)
395 text = text[:match_line.start(1)] + value.encode() + text[match_line.end(1):]
396 line = re.search(pattern, text).group(0)
397 llog.debug(f"Updating {old_line} to {line} in {edited_file}")
398 with open(edited_file, "wb") as file_handle:
399 file_handle.write(text)
402 ###############################################################################
404 ###############################################################################
406 CURRENT_TEST_STAGE = None
407 CURRENT_TEST_NAME = None
408 LOG_TAG = "%s/%s" % (os.path.basename(__file__), os.uname()[1])
412 def enter_test_stage(s):
413 """Group events into stages for status updates."""
414 global CURRENT_TEST_STAGE
415 CURRENT_TEST_STAGE = s
416 llog.info("Transitioning to test stage %s", s)
419 def progress(fmt, *args):
420 """Status updates that stand out among the log noise."""
421 if isinstance(CURRENT_TEST_STAGE, str):
422 label = "/%s" % CURRENT_TEST_STAGE
425 name = CURRENT_TEST_NAME if isinstance(CURRENT_TEST_NAME, str) else ""
426 fmt, label = str(fmt), str(label) # yes
427 llog.info("[%s%s] %s" % (name, label, fmt), *args)
428 # TODO: this method is more dynamic
429 # llog.info("[%s%s] %s%s" % (LOG_TAG, "", LOG_INDENT*indent, fmt), *args)
432 # these methods serve as shorter names
434 """Short name for INFO logging."""
435 llog.info(fmt, *args)
439 """Short name for DEBUG logging."""
440 llog.debug(fmt, *args)
444 """Short name for ERROR logging."""
445 llog.error(fmt, *args)
449 """Short name for WARN logging."""
450 llog.error(fmt, *args)
453 # these methods use the format capability
454 def log(level, text, *args, **kwargs):
455 """Log at any level using format capability."""
456 llog.log(level, text.format(*args), **kwargs)
459 def info(text, *args, **kwargs):
460 """Log at INFO level using format capability."""
461 log(logging.INFO, text, *args, **kwargs)
464 def debug(text, *args, **kwargs):
465 """Log at DEBUG level using format capability."""
466 log(logging.DEBUG, text, *args, **kwargs)
469 def error(text, *args, **kwargs):
470 """Log at ERROR level using format capability."""
471 log(logging.ERROR, text, *args, **kwargs)
474 def warn(text, *args, **kwargs):
475 """Log at WARN level using format capability."""
476 log(logging.WARN, text, *args, **kwargs)