1 # The software in this package is distributed under the GNU General
2 # Public License version 2 (with a special exception described below).
4 # A copy of GNU General Public License (GPL) is included in this distribution,
5 # in the file COPYING.GPL.
7 # As a special exception, if other files instantiate templates or use macros
8 # or inline functions from this file, or you compile this file and link it
9 # with other works to produce a work based on this file, this file
10 # does not by itself cause the resulting work to be covered
11 # by the GNU General Public License.
13 # However the source code for this file must still be made available
14 # in accordance with section (3) of the GNU General Public License.
16 # This exception does not invalidate any other reasons why a work based
17 # on this file might be covered by the GNU General Public License.
19 # Copyright (c) 2016-2022 Intra2net AG <info@intra2net.com>
22 store: implementations of CNF stores using varlink and `*et_cnf` binaries.
25 - CnfStore: the main store class that is implemented using the varlink API
26 - BinaryCnfStore: alternative store class implementation using the `set_cnf`
27 and `get_cnf` binaries
29 .. seealso:: Overview Diagram linked to from doc main page
31 .. codeauthor:: Intra2net
38 from .. import arnied_wrapper, arnied_api
39 from . import CnfBinary, Cnf, CnfList
41 log = logging.getLogger("pyi2ncommon.cnfvar.store")
45 """Main CNF store class that uses the varlink API."""
47 def __init__(self, backend_driver=arnied_api.Arnied):
49 Initialize this class.
51 :param backend_driver: driver to use to talk to the cnfvar backend
52 :type: :py:class:`arnied_api.Arnied`
54 self._driver = backend_driver
55 log.debug("Initialized CnfStore with driver `%s`", type(backend_driver))
57 def query(self, name=None, instance=None):
59 Query the CNF store and return a list of parsed CNFs.
61 :param str name: optional name of the CNFs to query
62 :param instance: optional CNF instance
63 :type instance: str or int
64 :returns: list of parsed CNF values
65 :rtype: :py:class:`CnfList`
69 user_cnfs = store.query("USER")
71 log.debug("Querying CnfStore with name=%s and instance=%s via arnied API",
74 # the API only expects an object if there is a name (and it's not case-insensitive)
75 query = arnied_api.GetCnfQuery(name.upper(), instance) if name else None
76 api_ret = self._driver.get_cnf(query)
78 # NOTE: logging all output here would result in huge lines when querying
79 # all variables via `store.query()`
80 log.debug("Arnied API returned %d cnfvars", len(api_ret.vars))
81 return CnfList.from_api_structure(api_ret.vars)
83 def commit(self, cnf, fix_problems=False):
85 Update or insert CNF variables from a list.
87 :param cnf: CNF instance or list of CNFs to update or insert
88 :type cnf: :py:class:`Cnf` or :py:class:`CnfList`
89 :raises: :py:class:`CommitException` if the arnied API complains
90 :param bool fix_problems: whether to automatically fix errors in the vars
92 .. note:: you can mix variables to insert and variables to update
93 in the same list as the system should handle it nicely
97 user_cnf = store.query("USER")\
98 .single_with_value("joe")
99 user_cnf.add_child("user_group_member_ref", "3")
100 store.commit(user_cnf)
102 cnf = self._cnf_or_list(cnf, operation="commit")
103 self._autofix_instances(cnf)
104 log.debug("Committing variables via arnied API:\n%s", cnf)
106 arnied_cnfs = cnf.to_api_structure()
107 self._do_commit(cnf, arnied_cnfs, fix_problems=fix_problems)
109 def delete(self, cnf, fix_problems=False):
111 Delete a list of top-level CNF variables.
113 :param cnf: a single CNF value or a list of values
114 :type cnf: :py:class:`CnfList` or :py:class:`Cnf`
115 :param bool fix_problems: whether to automatically fix errors in the vars
119 user_cnfs = store.query("USER")\
120 .where(lambda c: c.value in ["joe", "jane"])
121 store.delete(user_cnfs)
123 cnf = self._cnf_or_list(cnf, operation="delete")
124 if any((c.parent is not None for c in cnf)):
125 raise RuntimeError("Calling delete is only supported on top-level CNF variables")
126 log.debug("Deleting variables via arnied API:\n%s", cnf)
128 arnied_cnfs = cnf.to_api_structure()
129 for c in arnied_cnfs:
132 self._do_commit(cnf, arnied_cnfs, fix_problems=fix_problems)
134 def _autofix_instances(self, cnfs):
136 Auto-assign a valid instance value to all top-level vars in a list.
138 :param cnfs: list of cnfvars to fix
139 :type cnfs: :py:class`CnfList`
141 When the instance of a cnfvar is -1, the backend initializes it
142 automatically. However it starts on 1, whereas many variables are not
143 allowed to start on one (those that are meant to be unique, f.e.).
144 This method can be used in child classes to use an alternative scheme,
145 however for performance reasons the base API class uses the default and
146 relies on the cnfvar backend to do this job.
149 def _do_commit(self, original_cnfs, arnied_cnfs, fix_problems=False):
151 Set cnfvars and commit changes.
153 :param original_cnfs: list of CNFs to print in case of errors
154 :type original_cnfs: :py:class:`CnfList`
155 :param arnied_cnfs: list of cnfvars to pass to the arnied API
156 :type arnied_cnfs: [:py:class:`arnied_api.CnfVar`]
157 :param bool fix_problems: whether to automatically fix errors in the vars
158 :raises: :py:class:`CommitException` if the arnied API complains
161 ret = self._driver.set_commit_cnf(vars=arnied_cnfs, username=None,
163 fix_commit=fix_problems)
164 except arnied_api.CnfCommitError as ex:
165 # fatal errors, we will handle it in our custom exception
169 for r in ret.results:
170 # message can contain HTML escape codes
171 msg = html.unescape(r.result_msg)
173 # `result_type` is defined as int in the varlink API,
174 # but the meaning of the codes are:
175 # enum result_type { OK=0, WARN=1, FAIL_TEMP=2, FAIL=3 }
176 if r.result_type == 1:
177 log.debug("Warning in `` %s,%s: \"%s\" ``: {msg} (code=%s)",
178 r.name, r.instance, r.data, r.result_type)
179 elif r.result_type > 1:
180 errors.append(f"Error in `` {r.name},{r.instance}: \"{r.data}\" ``"
184 log.debug("Error sending variables:\n%s", arnied_cnfs)
185 raise CommitException(original_cnfs, "\n".join(errors))
187 self._wait_for_generate()
189 def _wait_for_generate(self, timeout=300):
191 Wait for the 'generate' program to end.
193 :param int timeout: program run timeout
194 :raises: :py:class:`TimeoutError` if the program did not finish on time
196 def scheduled_or_running(progname):
197 ret = self._driver.is_scheduled_or_running(progname)
198 return ret.status in [arnied_api.ProgramStatus.Scheduled,
199 arnied_api.ProgramStatus.Running]
201 def wait_for_program(progname):
202 log.debug("Waiting for `%s` to be running", progname)
204 if scheduled_or_running(progname):
205 # if is running or scheduled, break to wait for completion
209 # after trying and retrying, program is not scheduled nor
210 # running, so it is safe to assume it has already executed
213 # program running or scheduled, wait
214 log.debug("Waiting for `%s` to finish", progname)
215 for _ in range(0, timeout):
216 if not scheduled_or_running(progname):
217 # finished executing, bail out
220 raise TimeoutError(f"Program `{progname}` did not end in time")
222 wait_for_program("GENERATE")
223 wait_for_program("GENERATE_OFFLINE")
225 def _cnf_or_list(self, cnf, operation) -> CnfList:
227 Validate and wrap a CNF value into a list.
229 :param cnf: a single CNF value or a list of values
230 :type cnf: :py:class:`CnfList` or :py:class:`Cnf`
231 :param str operation: name of the operation that is being done
232 :raises: :py:class:`TypeError` if the type of the CNF object
233 is neither a list nor a CNF value
234 :returns: wrapped CNF value
235 :rtype: :py:class:`CnfList`
237 if isinstance(cnf, Cnf):
239 elif not isinstance(cnf, CnfList):
240 raise TypeError(f"Cannot {operation} value(s) of type `{type(cnf)}`")
244 class BinaryCnfStore(CnfStore):
245 """Implementation of the CNF store that uses `get_cnf` and `set_cnf`."""
247 #: how much to wait for arnied to report running
250 def __init__(self, backend_driver=CnfBinary):
252 Initialize this class.
254 :param backend_driver: driver to use to talk to the cnfvar backend
255 :type: :py:class:`CnfBinary`
257 super().__init__(backend_driver=backend_driver)
258 log.debug("Initialized BinaryCnfStore with driver `%s`", type(backend_driver))
260 def query(self, name=None, instance=None):
262 Query the CNF store and return a list of parsed CNFs.
264 :param str name: optional name of the CNFs to query
265 :param instance: optional CNF instance
266 :type instance: str or int
267 :returns: list of parsed CNF values
268 :rtype: :py:class:`CnfList`
272 user_cnfs = store.query("USER")
274 log.debug("Querying BinaryCnfStore with name=%s and instance=%s",
276 output = self._driver.get_cnf(name, instance=instance)
279 # otherwise cnfvar raises a generic Malformed exception
282 return CnfList.from_cnf_string(output)
284 def commit(self, cnf, fix_problems=False):
286 Update or insert CNF variables from a list.
288 :param cnf: CNF instance or list of CNFs to update or insert
289 :type cnf: :py:class:`Cnf` or :py:class:`CnfList`
290 :param bool fix_problems: whether to automatically fix errors in the vars
292 .. note:: you can mix variables to insert and variables to update
293 in the same list as the system should handle it nicely
297 user_cnf = store.query("USER")\
298 .single_with_value("joe")
299 user_cnf.add_child("user_group_member_ref", "3")
300 store.commit(user_cnf)
302 cnf = self._cnf_or_list(cnf, operation="commit")
303 self._autofix_instances(cnf)
305 # set_cnf is demanding on lineno's
307 log.debug("Committing variables via binaries:\n%s", cnf)
309 self._call_arnied(arnied_wrapper.verify_running, timeout=self.ARNIED_TIMEOUT)
311 self._driver.set_cnf(input_str=str(cnf), fix_problems=fix_problems)
312 except subprocess.CalledProcessError as ex:
313 raise CommitException(cnf, ex.stderr) from None
314 self._call_arnied(arnied_wrapper.wait_for_generate)
316 def delete(self, cnf, fix_problems=False):
318 Delete a list of top-level CNF variables.
320 :param cnf: a single CNF value or a list of values
321 :type cnf: :py:class:`CnfList` or :py:class:`Cnf`
322 :param bool fix_problems: whether to automatically fix errors in the vars
326 user_cnfs = store.query("USER")\
327 .where(lambda c: c.value in ["joe", "jane"])
328 store.delete(user_cnfs)
330 cnf = self._cnf_or_list(cnf, operation="delete")
331 if any((c.parent is not None for c in cnf)):
332 raise RuntimeError("Calling delete is only supported on top-level CNF variables")
334 # set_cnf is demanding on lineno's
336 log.debug("Deleting variables via binaries:\n%s", cnf)
338 self._call_arnied(arnied_wrapper.verify_running, timeout=self.ARNIED_TIMEOUT)
340 self._driver.set_cnf(input_str=str(cnf), delete=True, fix_problems=fix_problems)
341 except subprocess.CalledProcessError as ex:
342 raise CommitException(cnf, ex.stderr) from None
343 self._call_arnied(arnied_wrapper.wait_for_generate)
345 def _call_arnied(self, fn, *args, **kwargs):
347 Simple proxy around the arnied to be overwritten in child classes.
349 :param fn: function to invoke in the arnied wrapper
350 :type fn: py:function
351 :param args: arguments to be passed to function
352 :param kwargs: named arguments to be passed to function
354 return fn(*args, **kwargs)
357 #: pattern for more verbose error message for :py:class:`CommitException`
358 COMMIT_EXCEPTION_MESSAGE = """\
359 Error committing CNF variables!
360 ----------------------------
363 ----------------------------
369 class CommitException(Exception):
370 """Custom exception for commit errors."""
372 def __init__(self, cnfvars, msg):
374 Initialize this exception.
376 :param cnfvars: list of CNF variables that would be committed
377 :type cnfvars: CnfList
378 :param str msg: error message
380 super().__init__(msg)
381 self.message = COMMIT_EXCEPTION_MESSAGE.format(cnfvars=cnfvars, msg=msg)