1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
24 ------------------------------------------------------
25 Read / write / merge guest cnf var sets, even on host.
27 Copyright: Intra2net AG
31 ------------------------------------------------------
33 This module can be viewed as a convenience-wrapper around module
34 :py:mod:`cnfvar`. It uses many of its function but provides some higher-level
35 interfaces, most of all class :py:class:`SimpleCnf`. It is completely
36 independent of the :py:mod:`cnfline` package and its included subclasses
37 (modules in `shared.cnfline`, starting with ``build_`` and ``configure_``).
39 Class :py:class:`SimpleCnf` represents a hierarchical set of conf vars and
40 provides functions to deal with that hierarchy. Under the hood, all functions
41 here (and probably also in :py:mod:`cnfvar`) work with conf vars represented as
42 dictionaries and lists thereof. Conf var dicts have keys `number`, `varname`,
43 `instance`, `data`, `comment` and possibly `parent` and/or `children`.
44 `varname` is a regular lower-case string, `data` is a (utf8) string, `comment`
45 is usually None, `number`, `parent` and `instance` are int. If a conf var has
46 children, then this is a list of conf var dicts. `parent` is only present if a
47 conf-var is a child. Several conf vars, if not wrapped in a
48 :py:class:`SimpleCnf`, appear either as simple list of conf var dicts or as a
49 dict with a single key `cnf` whose value is a list of conf var dicts. (Function
50 :py:func:`get_cnf` returns the former given the latter).
52 .. todo:: Exceptions Invalid[Json]Cnf are used inconsistently (e.g. check type
53 of function arguments `-->` should be ValueError) and difference
54 between them is unclear. Also name differs only in case from
55 :py:class:`cnfvar.InvalidCNF`
58 ------------------------------------------------------
66 log = logging.getLogger('pyi2ncommon.simple_cnf')
68 from . import arnied_wrapper
72 ###############################################################################
74 ###############################################################################
76 #: timeout for copying temporary config files to VM objects (seconds)
77 COPY_FILES_TIMEOUT = 15
79 #: additional location of configuration files
80 ADD_CNFFILE_PATH = "/tmp/configs"
83 ###############################################################################
85 ###############################################################################
88 class InvalidCnf(Exception):
89 """Exception that indicates a general problem with conf var processing."""
91 def __init__(self, m):
92 """Create an invalid config exception."""
93 msg = "Invalid CNF_VAR: %s" % m
94 super(InvalidCnf, self).__init__(msg)
99 """Get a string version of the exception message."""
100 return "%s %s" % (self.pfx, self.msg)
103 class InvalidJsonCnf(InvalidCnf):
104 """Exception that indicates a general problem with conf var processing."""
106 def __init__(self, m):
107 """Create an invalid JSON config exception."""
108 super(InvalidJsonCnf, self).__init__(m)
109 self.pfx = "[CNF:JSON]"
112 ###############################################################################
113 # auxiliary functions
114 ###############################################################################
119 "Convert" a config dict to a list of conf var dicts.
121 This just removes the top-level 'cnf' key and returns its value.
123 :param cnf: config dictionary
124 :type cnf: {str, [dict]}
125 :returns: list of cnf var dicts
126 :rtype: [{str, int or str or None}]
127 :raises: :py:class:`InvalidJsonCnf` if there is no `cnf` field found
129 cnf_vars = cnf.get("cnf")
131 raise InvalidJsonCnf("toplevel \"cnf\" field required")
137 Get a (quite) safe temporary file name for config file.
139 :returns: temporary file name
143 file_handle, file_name = tempfile.mkstemp(prefix="simple_%d_" % int(now),
145 os.close(file_handle)
150 def set_values(cnf_vars, replacements):
152 Recursively replace values in configuration
154 Works in-place, meaning that no new configuration is created and returned
155 but instead argument `cnf_vars` is modified (and nothing returned).
157 :param cnf_vars: config where replacements are to be made
158 :type cnf_vars: [{str, int or str or None}] or {str, [dict]}
159 :param replacements: what to replace and what to replace it with
160 :type replacements: {str, str} or [(str, str)]
161 :raises: :py:class:`InvalidJsonCnf` if cnf_vars is neither dict or list
163 # determine set replace_me of keys to replace and function get that returns
164 # value for key or empty string if key not in replacements
167 if isinstance(replacements, dict):
168 replace_me = set(k.lower() for k in replacements.keys())
169 get = lambda var: str(replacements.get(var, "")) # pylint: disable=function-redefined
170 elif isinstance(replacements, list):
171 replace_me = set(r[0].lower() for r in replacements)
173 def get(var): # pylint: disable=function-redefined
174 """Get replacement value for given variable name."""
176 return str(next(r[1] for r in replacements if r[0] == var))
177 except StopIteration:
180 raise TypeError("replacements must be dictionary or key-value list")
182 # check type of arg "cnf_vars"
183 if isinstance(cnf_vars, dict):
184 cnf_vars = cnf_vars["cnf"] # operate on the var list
185 if not isinstance(cnf_vars, list):
186 raise InvalidJsonCnf("ill-formed CNF_VAR: expected list, got %s (%s)"
187 % (type(cnf_vars), cnf_vars))
190 """Internal recursive function to replace values."""
192 varname = var["varname"].lower()
193 if varname in replace_me:
194 var["data"] = str(get(varname))
195 children = var.get("children", None)
196 if children is not None:
199 # apply function on complete cnf_vars
203 def lookup_cnf_file(fname):
205 Searches for config file with given name in default locations.
207 :param str fname: file name of config file (without path)
208 :returns: first existing config file found in default locations
210 :raises: :py:class:`IOError` if no such config file was found
212 locations = [arnied_wrapper.SRC_CONFIG_DIR, ADD_CNFFILE_PATH]
213 for path in locations:
214 fullpath = os.path.join(path, fname)
215 if os.path.isfile(fullpath):
217 raise IOError("config file %s does not exist in any of the readable "
218 "locations %s" % (fname, locations))
221 ###############################################################################
223 ###############################################################################
226 class SimpleCnf(object):
228 Representation of hierarchical configuration of variables.
230 Based on C++ `cnf_vars` as visualized by *get_cnf*.
232 Internal data representation: list of conf var dicts; see module doc for
236 def __init__(self, cnf=None):
238 Creates a simple configuration.
240 Does not check whether given cnf list contains only valid data.
241 Does not recurse into dicts.
243 :param cnf: initial set of conf var data (default: None = empty conf)
244 :type cnf: list or anything that :py:func:`get_cnf` can read
248 elif isinstance(cnf, list):
250 elif isinstance(cnf, dict):
251 self.__cnfvars = get_cnf(cnf)
253 raise InvalidCnf ("cannot handle %s type inputs" % type (cnf))
255 def _find_new_number(self, cnf_vars):
256 """Recursive helper function to find new unique (line) number."""
259 new_numbers = [1, ] # in case cnf_vars is empty
260 for cnf_var in cnf_vars:
261 new_numbers.append(cnf_var['number'] + 1)
263 new_numbers.append(self._find_new_number(cnf_var['children']))
266 return max(new_numbers) # this is max(all numbers) + 1
268 def _find_new_instance(self, varname):
270 Find an instance number for variable with non-unique varname.
272 Will only check on top level, is not recursive.
274 :param str varname: name of conf var; will be converted to lower-case
275 :returns: instance number for which there is no other conf var of same
276 name (0 if there is not other conf var with that name)
280 varname = varname.lower()
281 for entry in self.__cnfvars:
282 if entry['varname'] == varname:
283 result = max(result, entry['number']+1)
286 def add(self, varname, data='', number=None, instance=None, children=None):
288 Add a cnf var to config on top level.
290 :param str varname: name of conf var; only required arg; case ignored
291 :param str data: conf var's value
292 :param int number: line number of that conf var; if given as None
293 (default) the function looks through config to find
294 a new number that is not taken; must be positive!
295 Value will be ignored if children are given.
296 :param int instance: Instance of the new conf var or None (default).
297 If None, then function looks through config to
298 find a new unique instance number
299 :param children: child confs for given conf var. Children's parent
300 and line attributes will be set in this function
301 :type children: :py:class:`SimpleCnf`
304 instance = self._find_new_instance(varname)
306 number = self._find_new_number(self.__cnfvars) # need top number
308 for child in children:
309 new_dict = child.get_single_dict()
310 new_dict['parent'] = number
311 new_children.append(new_dict)
312 cnfvar.renumber_vars({'cnf':new_children}, number)
313 children = new_children
315 number = self._find_new_number(self.__cnfvars)
317 new_var = dict(varname=varname.lower(), data=data,
318 number=number, comment=None, instance=instance)
320 new_var['children'] = children # only add if non-empty
321 self.__cnfvars.append(new_var)
324 def add_single(self, varname, data=u'', number=None):
326 Add a single cnf var to config on top level.
330 return self.add (varname, data=data, number=number)
333 def append_file_generic(self, reader, cnf, replacements=None):
335 Append conf var data from file.
337 If `replacements` are given, calls :py:meth:`set_values` with these
338 before adding values to config.
340 :param cnf: file name or dictionary of conf vars
341 :type cnf: str or {str, int or str or None}
342 :param replacements: see help in :py:meth:`set_values`
344 log.info("append CNF_VARs from file")
346 if callable(reader) is False:
347 raise TypeError("append_file_generic: reader must be callable, "
348 "not %s" % type(reader))
349 if isinstance(cnf, dict):
350 new_vars = get_cnf(cnf)
351 elif isinstance(cnf, str):
352 fullpath = lookup_cnf_file(cnf)
353 with open(fullpath, "rb") as chan:
354 cnfdata = chan.read()
355 tmp = reader(cnfdata)
356 new_vars = get_cnf(tmp)
358 raise InvalidCnf("Cannot append object \"%s\" of type \"%s\"."
361 if replacements is not None:
362 set_values(new_vars, replacements)
364 self.__cnfvars.extend(new_vars)
366 def append_file(self, cnf, replacements=None):
367 """Append conf var data from file."""
368 return self.append_file_generic(cnfvar.read_cnf, cnf,
369 replacements=replacements)
371 def append_file_json(self, cnf, replacements=None):
372 """Append conf var data from json file."""
373 return self.append_file_generic(cnfvar.read_cnf_json, cnf,
374 replacements=replacements)
376 def append_guest_vars(self, vm=None, varname=None, replacements=None):
378 Append content from machine's "real" config to this object.
380 Runs `get_cnf -j [varname]` on local host or VM (depending on arg
381 `vm`), converts output and appends it to this objects' conf var set.
382 If replacements are given, runs :py:meth:`set_values`, first.
384 :param vm: a guest vm or None to run on local host
385 :type vm: VM object or None
386 :param str varname: optional root of conf vars to append. If given as
387 None (default), append complete conf
388 :param replacements: see help in :py:meth:`set_values`
390 cnf = arnied_wrapper.get_cnfvar(varname=varname, vm=vm)
391 new_vars = get_cnf(cnf)
393 log.info("apply substitutions to extracted CNF_VARs")
394 if replacements is not None:
395 set_values(new_vars, replacements)
397 current = self.__cnfvars
398 current.extend(new_vars)
400 def save(self, filename=None):
402 Saves this object's configuration data to a file.
404 The output file's content can be interpreted by `set_cnf -j`.
406 :param str filename: name of file to write config to; if None (default)
407 the config will be written to a temporary file
408 :returns: filename that was written to
411 log.info("save configuration")
412 current = self.__cnfvars
414 raise InvalidCnf("No variables to write.")
417 # create temporary filename
418 filename = arnied_wrapper.generate_config_path(dumped=True)
420 with open(filename, 'w') as out:
421 cnfvar.output_json({"cnf": current}, out, renumber=True)
425 def apply(self, vm=None, renumber=True):
427 Apply object's config on VM or local host.
429 Runs a `set_cnf` with complete internal config data, possibly waits for
430 generate to finish afterwards.
432 :param vm: a guest vm or None to apply on local host
433 :type vm: VM object or None
434 :param bool renumber: re-number conf vars before application
436 current = self.__cnfvars
438 log.info("enforce consistent CNF_LINE numbering")
439 cnfvar.renumber_vars(current)
440 log.info("inject configuration %s" % "into guest" if vm else "in place")
441 arnied_wrapper.set_cnf_dynamic({"cnf": current},
442 config_file=gen_tmpname(), vm=vm)
446 Get a config in json format, ready for `set_cnf -j`.
448 :returns: config in json format
451 return cnfvar.dump_json_string({"cnf": self.__cnfvars}, renumber=True)
453 def pretty_print(self, print_func=None):
455 Get a string representation of this simple_cnf that is human-readable
457 Result is valid json with nice line breaks and indentation but not
458 renumbered (so may not be fit for parsing)
460 for line in json.dumps({"cnf": self.__cnfvars}, check_circular=False,
461 indent=4, sort_keys=True).splitlines():
462 if print_func is None:
469 Return an iterator over the contents of this simple cnf.
471 The iteration might not be ordered by line number nor entry nor
472 anything else. No guarantees made!
474 The entries returned by the iterator are :py:class:`SimpleCnf`.
478 for cnf_list in iter(my_cnf['PROXY_ACCESSLIST']):
479 print('checking proxy list {0} with {1} children'
480 .format(cnf_list.get_value(), len(cnf_list)))
482 # self.__cnfvars is a list of dicts, each with the same fields
483 for dict_entry in self.__cnfvars:
484 yield SimpleCnf([dict_entry, ])
486 def __getitem__(self, item):
488 Called by `cnf['key']` or `cnf[line_number]`; returns subset of cnf.
490 Processing time is O(n) where n is the number of top-level entries in
496 all.append_guest_vars()
497 len(all) # --> probably huge
498 len(all['user']) # should give the number of users
500 # should result in the same as all['user']:
502 users.append_guest_vars(varname='user')
504 :param item: line number or value to specify a cnf subset;
505 if string value, will be converted to lower case
506 :type item: int or str
507 :returns: another simple cnf that contains a subset of this simple cnf
508 :rtype: :py:class:`SimpleCnf`
510 .. seealso:: method :py:func:`get` (more general than this)
512 # determine whether arg 'item' is a key name or a line number
513 if isinstance(item, int): # is line number
515 else: # assume key name
519 # search all entries for matches
520 results = [dict_entry for dict_entry in self.__cnfvars
521 if dict_entry[dict_key] == item]
523 # convert result to a simple cnf
524 return SimpleCnf(results)
528 Get the number of top-level entries in cnf.
530 :returns: number of top-level entries in cnf
533 return len(self.__cnfvars)
535 def get(self, name=None, value=None, instance=None, line=None):
537 Get a subset of this config that matches ALL of given criteria.
539 For example, if :py:func:`get_cnf` contains the line
540 '1121 USER,1: "admin"', all of these examples will result in the same
543 cnf.get(name='user', value='admin')
544 cnf.get(name='user', instance=1)
545 cnf.get(name='user').get(value='admin')
548 :param str name: conf var name (key) or None to not use this criterion;
549 will be converted to lower case
550 :param str value: value of conf var or None to not use this criterion
551 :param int instance: instance number of value in a list (e.g. USERS)
552 or None to not use this criterion
553 :param int line: line number of None to not use this criterion
554 :returns: a simple cnf that contains only entries that match ALL of the
555 given criteria. If nothing matches the given criteria, an
556 empty simple cnf will be returned
557 :rtype: :py:class:`SimpleCnf`
559 .. seealso:: method :py:func:`__getitem__` (less general than this)
562 name_test = lambda test_val: True
565 name_test = lambda test_val: name == test_val['varname']
568 value_test = lambda test_val: True
571 value_test = lambda test_val: test_val['data'] == value
574 instance_test = lambda test_val: True
575 elif not isinstance(instance, int):
576 raise ValueError('expect int value for instance!')
578 instance_test = lambda test_val: instance == test_val['instance']
581 line_test = lambda test_val: True
582 elif not isinstance(line, int):
583 raise ValueError('expect int value for line number!')
585 line_test = lambda test_val: test_val['number'] == line
587 return SimpleCnf(list(entry for entry in self.__cnfvars
588 if name_test(entry) and value_test(entry)
589 and instance_test(entry) and line_test(entry)))
591 def get_children(self):
593 Get children of simple cnf of just 1 entry.
595 :returns: simple cnf children or an empty simple cnf if entry has
597 :rtype: :py:class:`SimpleCnf`
598 :raises: :py:class:`ValueError` if this simple cnf has more
602 raise ValueError('get_children only possible if len == 1 (is {0})!'
605 result = self.__cnfvars[0]['children']
614 return SimpleCnf(result)
618 Get a value of a simple cnf of just 1 entry.
620 :returns: str cnf value/data
622 :raises: :py:class:`ValueError` if this simple cnf has more
626 raise ValueError('get_value only possible if len == 1 (is {0})!'
628 return self.__cnfvars[0]['data']
630 def get_single_dict(self):
632 Get a dictionary of a simple cnf of just 1 entry.
634 :returns: dictionary of a simple cnf
635 :rtype: {str, int or str or None}
638 raise ValueError('get_single_dict only possible if len == 1 (is {0})!'
640 return self.__cnfvars[0]
642 def __eq__(self, other_cnf):
644 Determine wether `self` == `other_cnf`.
646 :param other_cnf: cnf to compare with
647 :type other_cnf: :py:class:`SimpleCnf`
648 :returns: whether all cnf var entries are equal
651 key_func = lambda cnf_var_entry: cnf_var_entry['number']
653 if isinstance (other_cnf, SimpleCnf) is False:
656 return sorted(self.__cnfvars, key=key_func) \
657 == sorted(other_cnf.__cnfvars, key=key_func) # pylint: disable=protected-access