1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2022 Intra2net AG <info@intra2net.com>
22 store: implementations of CNF stores using varlink and `*et_cnf` binaries.
25 - CnfStore: the main store class that is implemented using the varlink API
26 - BinaryCnfStore: alternative store class implementation using the `set_cnf`
27 and `get_cnf` binaries
29 .. seealso:: Overview Diagram linked to from doc main page
31 .. codeauthor:: Intra2net
38 from .. import arnied_wrapper, arnied_api
39 from . import CnfBinary, Cnf, CnfList
41 log = logging.getLogger("pyi2ncommon.cnfvar.store")
45 """Main CNF store class that uses the varlink API."""
47 def __init__(self, backend_driver=arnied_api.Arnied):
49 Initialize this class.
51 :param backend_driver: driver to use to talk to the cnfvar backend
52 :type: :py:class:`arnied_api.Arnied`
54 self._driver = backend_driver
55 log.debug(f"Initialized cnf store with driver `{backend_driver.__name__}`")
57 def query(self, name=None, instance=None):
59 Query the CNF store and return a list of parsed CNFs.
61 :param str name: optional name of the CNFs to query
62 :param instance: optional CNF instance
63 :type instance: str or int
64 :returns: list of parsed CNF values
65 :rtype: :py:class:`CnfList`
69 user_cnfs = store.query("USER")
71 log.debug("Querying CnfStore with name=%s and instance=%s via arnied API",
74 # the API only expects an object if there is a name (and it's not case-insensitive)
75 query = arnied_api.GetCnfQuery(name.upper(), instance) if name else None
76 api_ret = self._driver.get_cnf(query)
78 # NOTE: logging all output here would result in huge lines when querying
79 # all variables via `store.query()`
80 log.debug("Arnied API returned %d cnfvars", len(api_ret.vars))
81 return CnfList.from_api_structure(api_ret.vars)
83 def commit(self, cnf, fix_problems=False):
85 Update or insert CNF variables from a list.
87 :param cnf: CNF instance or list of CNFs to update or insert
88 :type cnf: :py:class:`Cnf` or :py:class:`CnfList`
89 :raises: :py:class:`CommitException` if the arnied API complains
90 :param bool fix_problems: whether to automatically fix errors in the vars
92 .. note:: you can mix variables to insert and variables to update
93 in the same list as the system should handle it nicely
97 user_cnf = store.query("USER")\
98 .single_with_value("joe")
99 user_cnf.add_child("user_group_member_ref", "3")
100 store.commit(user_cnf)
102 cnf = self._cnf_or_list(cnf, operation="commit")
103 self._autofix_instances(cnf)
104 log.debug("Committing variables via arnied API:\n%s", cnf)
106 arnied_cnfs = cnf.to_api_structure()
107 self._do_commit(cnf, arnied_cnfs, fix_problems=fix_problems)
109 def delete(self, cnf, fix_problems=False):
111 Delete a list of top-level CNF variables.
113 :param cnf: a single CNF value or a list of values
114 :type cnf: :py:class:`CnfList` or :py:class:`Cnf`
115 :param bool fix_problems: whether to automatically fix errors in the vars
119 user_cnfs = store.query("USER")\
120 .where(lambda c: c.value in ["joe", "jane"])
121 store.delete(user_cnfs)
123 cnf = self._cnf_or_list(cnf, operation="delete")
124 if any((c.parent is not None for c in cnf)):
125 raise RuntimeError("Calling delete is only supported on top-level CNF variables")
126 log.debug("Deleting variables via arnied API:\n%s", cnf)
128 arnied_cnfs = cnf.to_api_structure()
129 for c in arnied_cnfs:
132 self._do_commit(cnf, arnied_cnfs, fix_problems=fix_problems)
134 def _autofix_instances(self, cnfs):
136 Auto-assign a valid instance value to all top-level vars in a list.
138 :param cnfs: list of cnfvars to fix
139 :type cnfs: :py:class`CnfList`
141 When the instance of a cnfvar is -1, the backend initializes it
142 automatically. However, it starts on 1, whereas many variables are not
143 allowed to start on one (those that are meant to be unique, f.e.).
144 This method can be used in child classes to use an alternative scheme,
145 however for performance reasons the base API class uses the default and
146 relies on the cnfvar backend to do this job.
148 ..todo:: This method compensates for limitations in production code that
149 might end up fixed up there deprecating our patching here.
152 def _do_commit(self, original_cnfs, arnied_cnfs, fix_problems=False):
154 Set cnfvars and commit changes.
156 :param original_cnfs: list of CNFs to print in case of errors
157 :type original_cnfs: :py:class:`CnfList`
158 :param arnied_cnfs: list of cnfvars to pass to the arnied API
159 :type arnied_cnfs: [:py:class:`arnied_api.CnfVar`]
160 :param bool fix_problems: whether to automatically fix errors in the vars
161 :raises: :py:class:`CommitException` if the arnied API complains
164 ret = self._driver.set_commit_cnf(vars=arnied_cnfs, username=None,
166 fix_commit=fix_problems)
167 except arnied_api.CnfCommitError as ex:
168 # fatal errors, we will handle it in our custom exception
172 for r in ret.results:
173 # message can contain HTML escape codes
174 msg = html.unescape(r.result_msg)
176 # `result_type` is defined as int in the varlink API,
177 # but the meaning of the codes are:
178 # enum result_type { OK=0, WARN=1, FAIL_TEMP=2, FAIL=3 }
179 if r.result_type == 1:
180 log.debug("Warning in `` %s,%s: \"%s\" ``: {msg} (code=%s)",
181 r.name, r.instance, r.data, r.result_type)
182 elif r.result_type > 1:
183 errors.append(f"Error in `` {r.name},{r.instance}: \"{r.data}\" ``"
187 log.debug("Error sending variables:\n%s", arnied_cnfs)
188 raise CommitException(original_cnfs, "\n".join(errors))
190 self._wait_for_generate()
192 def _wait_for_generate(self, timeout=300):
194 Wait for the 'generate' program to end.
196 :param int timeout: program run timeout
197 :raises: :py:class:`TimeoutError` if the program did not finish on time
199 def scheduled_or_running(progname):
200 ret = self._driver.is_scheduled_or_running(progname)
201 return ret.status in [arnied_api.ProgramStatus.Scheduled,
202 arnied_api.ProgramStatus.Running]
204 def wait_for_program(progname):
205 log.debug("Waiting for `%s` to be running", progname)
207 if scheduled_or_running(progname):
208 # if is running or scheduled, break to wait for completion
212 # after trying and retrying, program is not scheduled nor
213 # running, so it is safe to assume it has already executed
216 # program running or scheduled, wait
217 log.debug("Waiting for `%s` to finish", progname)
218 for _ in range(0, timeout):
219 if not scheduled_or_running(progname):
220 # finished executing, bail out
223 raise TimeoutError(f"Program `{progname}` did not end in time")
225 wait_for_program("GENERATE")
226 wait_for_program("GENERATE_OFFLINE")
228 def _cnf_or_list(self, cnf, operation) -> CnfList:
230 Validate and wrap a CNF value into a list.
232 :param cnf: a single CNF value or a list of values
233 :type cnf: :py:class:`CnfList` or :py:class:`Cnf`
234 :param str operation: name of the operation that is being done
235 :raises: :py:class:`TypeError` if the type of the CNF object
236 is neither a list nor a CNF value
237 :returns: wrapped CNF value
238 :rtype: :py:class:`CnfList`
240 if isinstance(cnf, Cnf):
242 elif not isinstance(cnf, CnfList):
243 raise TypeError(f"Cannot {operation} value(s) of type `{type(cnf)}`")
247 class BinaryCnfStore(CnfStore):
248 """Implementation of the CNF store that uses `get_cnf` and `set_cnf`."""
250 #: how much to wait for arnied to report running
253 def __init__(self, backend_driver=CnfBinary):
255 Initialize this class.
257 :param backend_driver: driver to use to talk to the cnfvar backend
258 :type: :py:class:`CnfBinary`
260 super().__init__(backend_driver=backend_driver)
261 log.debug(f"Initialized binary cnf store with driver `{backend_driver.__name__}`")
263 def query(self, name=None, instance=None):
265 Query the CNF store and return a list of parsed CNFs.
267 :param str name: optional name of the CNFs to query
268 :param instance: optional CNF instance
269 :type instance: str or int
270 :returns: list of parsed CNF values
271 :rtype: :py:class:`CnfList`
275 user_cnfs = store.query("USER")
277 log.debug("Querying BinaryCnfStore with name=%s and instance=%s",
279 output = self._driver.get_cnf(name, instance=instance)
282 # otherwise cnfvar raises a generic Malformed exception
285 return CnfList.from_cnf_string(output)
287 def commit(self, cnf, fix_problems=False):
289 Update or insert CNF variables from a list.
291 :param cnf: CNF instance or list of CNFs to update or insert
292 :type cnf: :py:class:`Cnf` or :py:class:`CnfList`
293 :param bool fix_problems: whether to automatically fix errors in the vars
295 .. note:: you can mix variables to insert and variables to update
296 in the same list as the system should handle it nicely
300 user_cnf = store.query("USER")\
301 .single_with_value("joe")
302 user_cnf.add_child("user_group_member_ref", "3")
303 store.commit(user_cnf)
305 cnf = self._cnf_or_list(cnf, operation="commit")
306 self._autofix_instances(cnf)
308 # set_cnf is demanding on lineno's
310 log.debug("Committing variables via binaries:\n%s", cnf)
312 self._call_arnied(arnied_wrapper.verify_running, timeout=self.ARNIED_TIMEOUT)
314 self._driver.set_cnf(input_str=str(cnf), fix_problems=fix_problems)
315 except subprocess.CalledProcessError as ex:
316 raise CommitException(cnf, ex.stderr) from None
317 self._call_arnied(arnied_wrapper.wait_for_generate)
319 def delete(self, cnf, fix_problems=False):
321 Delete a list of top-level CNF variables.
323 :param cnf: a single CNF value or a list of values
324 :type cnf: :py:class:`CnfList` or :py:class:`Cnf`
325 :param bool fix_problems: whether to automatically fix errors in the vars
329 user_cnfs = store.query("USER")\
330 .where(lambda c: c.value in ["joe", "jane"])
331 store.delete(user_cnfs)
333 cnf = self._cnf_or_list(cnf, operation="delete")
334 if any((c.parent is not None for c in cnf)):
335 raise RuntimeError("Calling delete is only supported on top-level CNF variables")
337 # set_cnf is demanding on lineno's
339 log.debug("Deleting variables via binaries:\n%s", cnf)
341 self._call_arnied(arnied_wrapper.verify_running, timeout=self.ARNIED_TIMEOUT)
343 self._driver.set_cnf(input_str=str(cnf), delete=True, fix_problems=fix_problems)
344 except subprocess.CalledProcessError as ex:
345 raise CommitException(cnf, ex.stderr) from None
346 self._call_arnied(arnied_wrapper.wait_for_generate)
348 def _call_arnied(self, fn, *args, **kwargs):
350 Simple proxy around the arnied to be overwritten in child classes.
352 :param fn: function to invoke in the arnied wrapper
353 :type fn: py:function
354 :param args: arguments to be passed to function
355 :param kwargs: named arguments to be passed to function
357 return fn(*args, **kwargs)
360 #: pattern for more verbose error message for :py:class:`CommitException`
361 COMMIT_EXCEPTION_MESSAGE = """\
362 Error committing CNF variables!
363 ----------------------------
366 ----------------------------
372 class CommitException(Exception):
373 """Custom exception for commit errors."""
375 def __init__(self, cnfvars, msg):
377 Initialize this exception.
379 :param cnfvars: list of CNF variables that would be committed
380 :type cnfvars: CnfList
381 :param str msg: error message
383 super().__init__(msg)
384 self.message = COMMIT_EXCEPTION_MESSAGE.format(cnfvars=cnfvars, msg=msg)