import logging
log = logging.getLogger('pyi2ncommon.arnied_wrapper')
-from . import cnfvar_old
from . import sysmisc
return cnf_id
-def get_cnfvar(varname=None, instance=None, data=None, timeout=30, vm=None):
- """
- Invoke get_cnf and return a nested CNF structure.
-
- :param str varname: "varname" field of the CNF_VAR to look up
- :param instance: "instance" of that variable to return
- :type instance: int
- :param str data: "data" field by which the resulting CNF_VAR list should be filtered
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :returns: the resulting "cnfvar" structure or None if the lookup fails or the result could not be parsed
- :rtype: cnfvar option
- """
- wait_for_arnied(timeout=timeout, vm=vm)
- # firstly, build argv for get_cnf
- cmd = ["get_cnf", "-j"]
- if varname is not None:
- cmd.append("%s" % varname)
- if instance:
- cmd.append("%d" % instance)
- cmd_line = " ".join(cmd)
-
- # now invoke get_cnf
- result = run_cmd(cmd=cmd_line, vm=vm)
- (status, raw) = result.returncode, result.stdout
- if status != 0:
- log.info("error %d executing \"%s\"", status, cmd_line)
- log.debug(raw)
- return None
-
- # reading was successful, attempt to parse what we got
- try:
- # The output from "get_cnf -j" is already utf-8. This contrast with
- # the output of "get_cnf" (no json) which is latin1.
- if isinstance(raw, bytes):
- raw = raw.decode("utf-8")
- cnf = cnfvar_old.read_cnf_json(raw)
- except TypeError as exn:
- log.info("error \"%s\" parsing result of \"%s\"", exn, cmd_line)
- return None
- except cnfvar_old.InvalidCNF as exn:
- log.info("error \"%s\" validating result of \"%s\"", exn, cmd_line)
- return None
-
- if data is not None:
- return cnfvar_old.get_vars(cnf, data=data)
-
- return cnf
-
-
-def get_cnfvar_id(varname, data, timeout=30, vm=None):
- """
- Similar to :py:func:`get_cnf_id` but uses :py:func:`get_cnfvar`.
-
- :param str varname: "varname" field of the CNF_VAR to look up
- :param str data: "data" field by which the resulting CNF_VAR list should be filtered
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :returns: the cnf id or -1 if no such cnf variable
- :rtype: int
- """
- wait_for_arnied(timeout=timeout, vm=vm)
- log.info("Extracting from arnied CNF_VAR %s with data %s",
- varname, data)
- cnf = get_cnfvar(varname=varname, data=data, vm=vm)
- variables = cnf["cnf"]
- if len(variables) == 0:
- log.info("CNF_VAR extraction unsuccessful, defaulting to -1")
- # preserve behavior
- return -1
- first_instance = int(variables[0]["instance"])
- log.info("CNF_VAR instance lookup yielded %d results, returning first value (%d)",
- len(variables), first_instance)
- return first_instance
-
-
def wait_for_generate(timeout=300, vm=None):
"""
Wait for the 'generate' program to complete.
the host. If these are absolute paths, they will be kept as is or
otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
the config files will be copied there as temporary files before applying.
+
+ ..todo:: The static method must be deprecated after we drop and convert
+ lots of use cases for it to dynamic only.
"""
log.info("Setting arnied configuration")
wait_for_arnied(timeout=timeout, vm=vm)
the host. If these are absolute paths, they will be kept as is or
otherwise will be searched for in `SRC_CONFIG_DIR`. If a vm is provided,
the config files will be copied there as temporary files before applying.
+
+ ..todo:: The semi-dynamic method must be deprecated after we drop and convert
+ lots of use cases for it to dynamic only.
"""
log.info("Performing semi-dynamic arnied configuration")
log.info("Semi-dynamic arnied configuration successful!")
-def set_cnf_dynamic(cnf, config_file=None, kind="cnf", timeout=30, vm=None):
- """
- Perform dynamic arnied configuration from fully generated config files.
-
- :param cnf: one key with the same value as *kind* and a list of cnfvars as value
- :type cnf: {str, str}
- :param config_file: optional user supplied filename
- :type config_file: str or None
- :param str kind: "json", "cnf", or "raw"
- :param int timeout: arnied run verification timeout
- :param vm: vm to run on if running on a guest instead of the host
- :type vm: :py:class:`virttest.qemu_vm.VM` or None
- :raises: :py:class:`ValueError` if `kind` is not an acceptable value
- :raises: :py:class:`ConfigError` if cannot apply file
-
- The config file might not be provided in which case a temporary file will
- be generated and saved on the host's `DUMP_CONFIG_DIR` of not provided as
- an absolute path. If a vm is provided, the config file will be copied there
- as a temporary file before applying.
- """
- if config_file is None:
- config_path = generate_config_path(dumped=True)
- elif os.path.isabs(config_file):
- config_path = config_file
- else:
- config_path = os.path.join(os.path.abspath(DUMP_CONFIG_DIR), config_file)
- generated = config_file is None
- config_file = os.path.basename(config_path)
- log.info("Using %s cnf file %s%s",
- "generated" if generated else "user-supplied",
- config_file, " on %s" % vm.name if vm is not None else "")
-
- # Important to write bytes here to ensure text is encoded with latin-1
- fd = open(config_path, "wb")
- try:
- SET_CNF_METHODS = {
- "raw": cnfvar_old.write_cnf_raw,
- "json": cnfvar_old.write_cnf_json,
- "cnf": cnfvar_old.write_cnf
- }
- SET_CNF_METHODS[kind](cnf, out=fd)
- except KeyError:
- raise ValueError("Invalid set_cnf method \"%s\"; expected \"json\" or \"cnf\""
- % kind)
- finally:
- fd.close()
- log.info("Generated config file %s", config_path)
-
- kind = "cnf" if kind != "json" else kind
- set_cnf([config_path], kind=kind, timeout=timeout, vm=vm)
-
-
def set_cnf_pipe(cnf, timeout=30, block=False):
"""
Set local configuration by talking to arnied via ``set_cnf``.
import json
-from .. import cnfvar_old, arnied_api
+from . import string
+from .. import arnied_api
#: value used to detect unspecified arguments
DEFAULT = object()
:returns: a list of cnfvars
:rtype: :py:class:`CnfList`
"""
- cnf_obj = cnfvar_old.read_cnf(data)
+ cnf_obj = string.read_cnf(data)
return CnfList.from_cnf_structure(cnf_obj)
@classmethod
--- /dev/null
+#!/usr/bin/env python
+#
+# The software in this package is distributed under the GNU General
+# Public License version 2 (with a special exception described below).
+#
+# A copy of GNU General Public License (GPL) is included in this distribution,
+# in the file COPYING.GPL.
+#
+# As a special exception, if other files instantiate templates or use macros
+# or inline functions from this file, or you compile this file and link it
+# with other works to produce a work based on this file, this file
+# does not by itself cause the resulting work to be covered
+# by the GNU General Public License.
+#
+# However the source code for this file must still be made available
+# in accordance with section (3) of the GNU General Public License.
+#
+# This exception does not invalidate any other reasons why a work based
+# on this file might be covered by the GNU General Public License.
+#
+# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
+
+"""
+string: functionality for parsing cnfvars from strings
+
+.. codeauthor:: Intra2net
+
+
+contents
+-------------------------------------------------------------------------------
+
+This module provides read and parse functionality for the Intra2net *CNF*
+format from strings and by extension cnf files.
+
+The input string takes one round-trip through the parsers and will error out on
+problematic lines. Thus, this module can also be used to syntax-check CNF data.
+
+Note that line numbers may be arbitrarily reassigned in the process. Of course,
+parent references and the relative ordering of lines will be preserved in this
+case.
+
+.. todo::
+ Decide on some facility for automatic fixup of line number values. The
+ internal representation is recursive so line numbers are not needed to
+ establish a variable hierarchy. They might as well be omitted from the json
+ input and only added when writing cnf. For the time being a lack of the
+ "number" field is interpreted as an error. Though it should at least
+ optionally be possible to omit the numbers entirely and have a function add
+ them after the fact. This might as well be added to :py:func:`is_cnf`
+ though it would be counterintuitive to have a predicate mutate its
+ argument. So maybe it could return a second argument to indicate a valid
+ structure that needs fixup or something like that.
+
+.. note::
+ The variable values of get_cnf seems to be encoded in latin1, and set_cnf
+ seems to assume latin1-encoded values (not var names). Function
+ :py:func:`read_cnf` converts this to unicode and other functions convert
+ unicode back to latin1.
+
+
+notes on Python 3 conversion
+-------------------------------------------------------------------------------
+
+Since the original *CNF* format assumes latin-1 encoded data pretty much
+exclusively, we preserve the original encoding while parsing the file.
+When assembling the data structures returned to the user, values are then
+converted to strings so they can be used naturally at the Python end.
+
+implementation
+-------------------------------------------------------------------------------
+"""
+
+import functools
+import sys
+import json
+import re
+import io
+
+
+###############################################################################
+# CONSTANTS
+###############################################################################
+
+
+CNF_FIELD_MANDATORY = set ([ "varname", "data", "instance" ])
+CNF_FIELD_OPTIONAL = set ([ "parent", "children", "comment", "number" ])
+CNF_FIELD_KNOWN = CNF_FIELD_MANDATORY | CNF_FIELD_OPTIONAL
+
+grab_parent_pattern = re.compile(b"""
+ ^ # match from start
+ \s* # optional spaces
+ \d+ # line number
+ \s+ # spaces
+ \((\d+)\) # parent
+ """,
+ re.VERBOSE)
+
+base_line_pattern = re.compile(b"""
+ ^ # match from start
+ \s* # optional spaces
+ (\d+) # line number
+ \s+ # spaces
+ ([A-Z][A-Z0-9_]*) # varname
+ \s* # optional spaces
+ , # delimiter
+ \s* # optional spaces
+ (-1|\d+) # instance
+ \s* # optional spaces
+ : # delimiter
+ \s* # optional spaces
+ \"( # quoted string (data)
+ (?: \\\" # (of escaped dquote
+ |[^\"])* # or anything not a
+ )\" # literal quote)
+ \s* # optional spaces
+ ( # bgroup
+ \# # comment leader
+ \s* # optional spaces
+ .* # string (comment)
+ )? # egroup, optional
+ $ # eol
+ """,
+ re.VERBOSE)
+
+child_line_pattern = re.compile(b"""
+ ^ # match from start
+ \s* # optional spaces
+ (\d+) # line number
+ \s+ # spaces
+ \((\d+)\) # parent
+ \s+ # spaces
+ ([A-Z][A-Z0-9_]*) # varname
+ \s* # optional spaces
+ , # delimiter
+ \s* # optional spaces
+ (-1|\d+) # instance
+ \s* # optional spaces
+ : # delimiter
+ \s* # optional spaces
+ \"([^\"]*)\" # quoted string (data)
+ \s* # optional spaces
+ ( # bgroup
+ \# # comment leader
+ \s* # optional spaces
+ .* # string (comment)
+ )? # egroup, optional
+ $ # eol
+ """,
+ re.VERBOSE)
+
+
+###############################################################################
+# HELPERS
+###############################################################################
+
+
+#
+# Sadly, the Intranator is still stuck with one leg in the 90s.
+#
+def to_latin1(s):
+ """Take given unicode str and convert it to a latin1-encoded `bytes`."""
+ return s.encode("latin-1")
+
+
+def from_latin1(s):
+ """Take given latin1-encoded `bytes` value and convert it to `str`."""
+ return s.decode("latin-1")
+
+
+#
+# Conversion functions
+#
+
+def marshal_in_number(number):
+ return int(number)
+
+
+def marshal_in_parent(parent):
+ return int(parent)
+
+
+def marshal_in_instance(instance):
+ return int(instance)
+
+
+def marshal_in_varname(varname):
+ return from_latin1(varname).lower()
+
+
+def marshal_in_data(data):
+ return from_latin1(data) if data is not None else ""
+
+
+def marshal_in_comment(comment):
+ return comment and from_latin1(comment[1:].strip()) or None
+
+
+#
+# Type checking
+#
+
+def is_string(s):
+ return isinstance(s, str)
+
+
+###############################################################################
+# EXCEPTIONS
+###############################################################################
+
+
+class InvalidCNF(Exception):
+
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return "Malformed CNF_VAR: \"%s\"" % self.msg
+
+
+class MalformedCNF(Exception):
+
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return "Malformed CNF file: \"%s\"" % self.msg
+
+
+###############################################################################
+# VALIDATION
+###############################################################################
+
+
+def is_valid(acc,
+ nested,
+ comment,
+ data,
+ instance,
+ number,
+ parent,
+ varname):
+ if varname is None:
+ raise InvalidCNF("CNF_VAR lacks a name.")
+ elif not is_string(varname):
+ raise InvalidCNF("Varname field of CNF_VAR \"%s\" is not a string."
+ % varname)
+ elif varname == "":
+ raise InvalidCNF("Varname field of CNF_VAR is the empty string.")
+
+ if comment is not None:
+ if not is_string(comment):
+ raise InvalidCNF("Comment field of CNF_VAR \"%s\" is not a string."
+ % varname)
+
+ if data is None:
+ raise InvalidCNF("Data field of CNF_VAR \"%s\" is empty."
+ % varname)
+ elif not is_string(data):
+ raise InvalidCNF("Data field of CNF_VAR \"%s\" is not a string."
+ % varname)
+
+ if instance is None:
+ raise InvalidCNF("Instance field of CNF_VAR \"%s\" is empty."
+ % varname)
+ elif not isinstance(instance, int):
+ raise InvalidCNF("Instance field of CNF_VAR \"%s\" is not an integer."
+ % varname)
+
+ if number is None:
+ raise InvalidCNF("Number field of CNF_VAR \"%s\" is empty."
+ % varname)
+ elif not isinstance(number, int):
+ raise InvalidCNF("Number field of CNF_VAR \"%s\" is not an integer."
+ % varname)
+ elif number < 1:
+ raise InvalidCNF("Number field of CNF_VAR \"%s\" must be positive, not %d."
+ % (varname, number))
+ else:
+ other = acc.get(number, None)
+ if other is not None: # already in use
+ raise InvalidCNF("Number field of CNF_VAR \"%s\" already used by variable %s."
+ % (varname, other))
+ acc[number] = varname
+
+ if nested is True:
+ if parent is None:
+ raise InvalidCNF("Parent field of nested CNF_VAR \"%s\" is empty."
+ % varname)
+ elif not isinstance(parent, int):
+ raise InvalidCNF("Parent field of CNF_VAR \"%s\" is not an integer."
+ % varname)
+ else:
+ if parent is not None:
+ raise InvalidCNF("Flat CNF_VAR \"%s\" has nonsensical parent field \"%s\"."
+ % (varname, parent))
+ return acc
+
+
+def is_cnf(root):
+ """
+ is_cnf -- Predicate testing "CNF_VAR-ness". Folds the :py:func:`is_valid`
+ predicate over the argument which can be either a well-formed CNF
+ dictionary or a list of CNF_VARs.
+
+ :type root: cnfvar or cnf list
+ :rtype: bool
+
+ Not that if it returns at all, ``is_cnf()`` returns ``True``. Any non
+ well-formed member of the argument will cause the predicate to bail out
+ with an exception during traversal.
+ """
+ cnf = cnf_root(root)
+ if cnf is None:
+ raise InvalidCNF(root)
+ return walk_cnf(cnf, False, is_valid, {}) is not None
+
+
+def is_cnf_var(obj):
+ """
+ Check whether a dictionary is a valid CNF.
+
+ :param dict obj: dictionary to check
+ :returns: True if the dictionary has all the mandatory fields and no
+ unknown fields, False otherwise
+ :rtype: bool
+ """
+ assert isinstance (obj, dict)
+
+ for f in CNF_FIELD_MANDATORY:
+ if obj.get(f, None) is None:
+ return False
+
+ for f in obj:
+ if f not in CNF_FIELD_KNOWN:
+ return False
+
+ return True
+
+
+###############################################################################
+# DESERIALIZATION
+###############################################################################
+#
+# CNF reader
+#
+# Parsing usually starts from the `read_cnf`, which accepts a string containing
+# the variables to parse in the same structure as returned by `get_cnf`.
+#
+# In the `prepare` function the string is split into lines, and a 3-element
+# tuple is built. The first (named `current`) and second (named `next`)
+# elements of this tuple are respectively the first and second non-empty lines
+# of the input, while the third is a list of the remaining lines. This tuple is
+# named `state` in the implementation below, and it is passed around during
+# parsing. The `get` and `peek` functions are used to easily retrieve the
+# `current` and `next` items from the "state".
+#
+# When we "advance" the state, we actually drop the "current" element,
+# replacing it with the "next", while a new "next" is popped from the list of
+# remaining lines. Parsing is done this way because we need to look ahead at
+# the next line -- if it is a child it needs to be appended to the `children`
+# property of the current line.
+#
+# Regular expressions are used to extract important information from the CNF
+# lines. Finally, once parsing is completed, a dictionary is returned. The dict
+# has the same structure as the serialized JSON output returned by
+# `get_cnf -j`.
+#
+
+
+def read_cnf(data):
+ """
+ Read cnf data from data bytes.
+
+ :param data: raw data
+ :type data: str or bytes
+ :return: the parsed cnf data
+ :rtype: {str, {str, str or int}}
+ """
+ if isinstance(data, str):
+ data = to_latin1(data)
+ state = prepare(data)
+ if state is None:
+ raise InvalidCNF("Empty input string.")
+
+ cnf = parse_cnf_root(state)
+ if is_cnf(cnf) is False:
+ raise TypeError("Invalid CNF_VAR.")
+ return {"cnf": cnf}
+
+
+def prepare(raw):
+ """
+ Build 3-element iterable from a CNF string dump.
+
+ :param raw: string content as returned by `get_cnf`
+ :type raw: bytes
+ :returns: 3-element tuple, where the first two elements are the first two
+ lines of the output and the third is a list containing the rest
+ of the lines in reverse.
+ :rtype: (str * str option * str list) option
+ """
+ lines = raw.splitlines()
+ lines.reverse()
+ try:
+ first = lines.pop()
+ except IndexError:
+ return None
+
+ try:
+ second = lines.pop()
+ except IndexError:
+ second = None
+
+ first = first.strip()
+ if first == b"":
+ return advance((first, second, lines))
+
+ return (first, second, lines)
+
+
+def advance(cns):
+ """
+ Pop the next line from the stream, advancing the tuple.
+
+ :param cns: a 3-element tuple containing two CNF lines and a list of the
+ remaining lines
+ :type cnd: (str, str, [str])
+ :returns: a new tuple with a new item popped from the list of lines
+ :rtype cnd: (str, str, [str])
+ """
+ current, next, stream = cns
+ if next is None: # reached end of stream
+ return None
+ current = next
+
+ try:
+ next = stream.pop()
+ next = next.strip()
+ except IndexError:
+ next = None
+
+ if current == "": # skip blank lines
+ return advance((current, next, stream))
+ return (current, next, stream)
+
+
+def get(cns):
+ """
+ Get the current line from the state without advancing it.
+
+ :param cns: a 3-element tuple containing two CNF lines and a list of the
+ remaining lines
+ :type cnd: (str, str, [str])
+ :returns: the CNF line stored as `current`
+ :rtype: str
+ """
+ current, _, _ = cns
+ return current
+
+
+def parse_cnf_root(state):
+ """
+ Iterate over and parse a list of CNF lines.
+
+ :param state: a 3-element tuple containing two lines and a list of the
+ remaining lines
+ :type state: (str, str, [str])
+ :returns: a list of parsed CNF variables
+ :rtype: [dict]
+
+ The function will parse the first element from the `state` tuple, then read
+ the next line to see if it is a child variable. If it is, it will be
+ appended to the last parsed CNF, otherwise top-level parsing is done
+ normally.
+ """
+ lines = []
+ current = get(state)
+ while state:
+ cnf_line = read_base_line(current)
+ if cnf_line is not None:
+ lines.append(cnf_line)
+ state = advance(state)
+ if state is None: # -> nothing left to do
+ break
+ current = get(state)
+ parent = get_parent(current) # peek at next line
+ if parent is not None: # -> recurse into children
+ (state, children, _parent) = parse_cnf_children(state, parent)
+ cnf_line["children"] = children
+ if state is None:
+ break
+ current = get(state)
+ else:
+ state = advance(state)
+ if state is None:
+ break
+ current = get(state)
+ return lines
+
+
+def parse_cnf_children(state, parent):
+ """
+ Read and parse child CNFs of a given parent until there is none left.
+
+ :param state: a 3-element tuple containing two lines and a list of the
+ remaining lines
+ :type state: (str, str, [str])
+ :param parent: id of the parent whose children we are looking for
+ :type parent: int
+ :returns: a 3-element tuple with the current state, a list of children of
+ the given parent and the parent ID
+ :rtype: (tuple, [str], int)
+
+ The function will recursively parse child lines from the `state` tuple
+ until one of these conditions is satisfied:
+
+ 1. the input is exhausted
+ 2. the next CNF line
+ 2.1. is a toplevel line
+ 2.2. is a child line whose parent has a lower parent number
+
+ Conceptually, 2.1 is a very similar to 2.2 but due to the special status of
+ toplevel lines in CNF we need to handle them separately.
+
+ Note that since nesting of CNF vars is achieved via parent line numbers,
+ lines with different parents could appear out of order. libcnffile will
+ happily parse those and still assign children to the specified parent:
+
+ ::
+ # set_cnf <<THATSALL
+ 1 USER,1337: "l33t_h4x0r"
+ 2 (1) USER_GROUP_MEMBER_REF,0: "2"
+ 4 USER,1701: "picard"
+ 5 (4) USER_GROUP_MEMBER_REF,0: "2"
+ 6 (4) USER_PASSWORD,0: "engage"
+ 3 (1) USER_PASSWORD,0: "hacktheplanet"
+ THATSALL
+ # get_cnf user 1337
+ 1 USER,1337: "l33t_h4x0r"
+ 2 (1) USER_GROUP_MEMBER_REF,0: "2"
+ 3 (1) USER_PASSWORD,0: "hacktheplanet"
+ # get_cnf user 1701
+ 1 USER,1701: "picard"
+ 2 (1) USER_GROUP_MEMBER_REF,0: "2"
+ 3 (1) USER_PASSWORD,0: "engage"
+
+ It is a limitation of ``cnfvar.py`` that it cannot parse CNF data
+ structured like the above example: child lists are only populated from
+ subsequent CNF vars using the parent number solely to track nesting levels.
+ The parser does not keep track of line numbers while traversing the input
+ so it doesn’t support retroactively assigning a child to anything else but
+ the immediate parent.
+ """
+ lines = []
+ current = get(state)
+ while True:
+ cnf_line = read_child_line(current)
+ if cnf_line is not None:
+ lines.append(cnf_line)
+ state = advance(state)
+ if state is None:
+ break
+ current = get(state)
+ new_parent = get_parent(current)
+ if new_parent is None:
+ # drop stack
+ return (state, lines, None)
+ if new_parent > parent:
+ # parent is further down in hierarchy -> new level
+ (state, children, new_parent) = \
+ parse_cnf_children (state, new_parent)
+ if state is None:
+ break
+ cnf_line["children"] = children
+ current = get(state)
+ new_parent = get_parent(current)
+ if new_parent is None:
+ # drop stack
+ return (state, lines, None)
+ if new_parent < parent:
+ # parent is further up in hierarchy -> pop level
+ return (state, lines, new_parent)
+ # new_parent == parent -> continue parsing on same level
+ return (state, lines, parent)
+
+
+def get_parent(line):
+ """
+ Extract the ID of the parent for a given CNF line.
+
+ :param str line: CNF line
+ :returns: parent ID or None if no parent is found
+ :rtype: int or None
+ """
+ match = re.match(grab_parent_pattern, line)
+ if match is None: # -> no parent
+ return None
+ return int(match.groups()[0])
+
+
+def read_base_line(line):
+ """
+ Turn one top-level CNF line into a dictionary.
+
+ :param str line: CNF line
+ :rtype: {str: Any}
+
+ This performs the necessary decoding on values to obtain proper Python
+ strings from 8-bit encoded CNF data.
+
+ The function only operates on individual lines. Argument strings that
+ contain data for multiple lines – this includes child lines of the current
+ CNF var! – will trigger a parsing exception.
+ """
+ if len(line.strip()) == 0:
+ return None # ignore empty lines
+ if line[0] == b"#":
+ return None # ignore comments
+
+ match = re.match(base_line_pattern, line)
+ if match is None:
+ raise MalformedCNF("Syntax error in line \"\"\"%s\"\"\"" % line)
+ number, varname, instance, data, comment = match.groups()
+ return {
+ "number" : marshal_in_number (number),
+ "varname" : marshal_in_varname (varname),
+ "instance" : marshal_in_instance (instance),
+ "data" : marshal_in_data (data),
+ "comment" : marshal_in_comment (comment),
+ }
+
+
+def read_child_line(line):
+ """
+ Turn one child CNF line into a dictionary.
+
+ :param str line: CNF line
+ :rtype: {str: Any}
+
+ This function only operates on individual lines. If the argument string is
+ syntactically valid but contains input representing multiple CNF vars, a
+ parse error will be thrown.
+ """
+ if len(line.strip()) == 0:
+ return None # ignore empty lines
+ if line[0] == "#":
+ return None # ignore comments
+
+ match = re.match(child_line_pattern, line)
+ if match is None:
+ raise MalformedCNF("Syntax error in child line \"\"\"%s\"\"\""
+ % from_latin1 (line))
+ number, parent, varname, instance, data, comment = match.groups()
+ return {
+ "number" : marshal_in_number (number),
+ "parent" : marshal_in_parent (parent),
+ "varname" : marshal_in_varname (varname),
+ "instance" : marshal_in_instance (instance),
+ "data" : marshal_in_data (data),
+ "comment" : marshal_in_comment (comment),
+ }
+
+
+###############################################################################
+# SERIALIZATION
+###############################################################################
+
+
+def cnf_root(root):
+ """
+ Extract a list of CNFs from a given structure.
+
+ :param root: list of CNFs or a CNF dictionary
+ :type root: [dict] or dict
+ :raises: :py:class:`TypeError` if no CNFs can be extracted
+ :returns: list with one or more CNF objects
+ :rtype: [dict]
+
+ Output varies depending on a few conditions:
+ - If `root` is a list, return it right away
+ - If `root` is a dict corresponding to a valid CNF value, return it wrapped
+ in a list
+ - If `root` is a dict with a `cnf` key containg a list (as the JSON
+ returned by `get_cnf -j`), return the value
+ - Otherwise, raise an error
+ """
+ if isinstance(root, list):
+ return root
+ if not isinstance(root, dict):
+ raise TypeError(
+ "Expected dictionary of CNF_VARs, got %s." % type(root))
+ if is_cnf_var(root):
+ return [root]
+ cnf = root.get("cnf", None)
+ if not isinstance(cnf, list):
+ raise TypeError("Expected list of CNF_VARs, got %s." % type(cnf))
+ return cnf
+
+
+###############################################################################
+# TRAVERSAL
+###############################################################################
+
+
+def walk_cnf(cnf, nested, fun, acc):
+ """
+ Depth-first traversal of a CNF tree.
+
+ :type cnf: cnf list
+ :type nested: bool
+ :type fun: 'a -> bool -> (cnf stuff) -> 'a
+ :type acc: 'a
+ :rtype: 'a
+
+ Executes ``fun`` recursively for each node in the tree. The function
+ receives the accumulator ``acc`` which can be of an arbitrary type as first
+ argument. The second argument is a flag indicating whether the current
+ CNF var is a child (if ``True``) or a parent var. CNF member fields are
+ passed via named optional arguments.
+ """
+ for var in cnf:
+ acc = fun(acc,
+ nested,
+ comment=var.get("comment", None),
+ data=var.get("data", None),
+ instance=var.get("instance", None),
+ number=var.get("number", None),
+ parent=var.get("parent", None),
+ varname=var.get("varname", None))
+ children = var.get("children", None)
+ if children is not None:
+ acc = walk_cnf(children, True, fun, acc)
+ return acc
+++ /dev/null
-#!/usr/bin/env python
-#
-# The software in this package is distributed under the GNU General
-# Public License version 2 (with a special exception described below).
-#
-# A copy of GNU General Public License (GPL) is included in this distribution,
-# in the file COPYING.GPL.
-#
-# As a special exception, if other files instantiate templates or use macros
-# or inline functions from this file, or you compile this file and link it
-# with other works to produce a work based on this file, this file
-# does not by itself cause the resulting work to be covered
-# by the GNU General Public License.
-#
-# However the source code for this file must still be made available
-# in accordance with section (3) of the GNU General Public License.
-#
-# This exception does not invalidate any other reasons why a work based
-# on this file might be covered by the GNU General Public License.
-#
-# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
-
-"""
-
-summary
--------------------------------------------------------------------------------
-Represent CNF_VARs as recursive structures.
-
-.. note:: DEPRECATED! Please do not extend this or add new uses of this module,
- use :py:mod:`pyi2ncommon.arnied_api` or :py:mod:`pyi2ncommon.cnfvar`
- instead!
-
-Copyright: 2014-2017 Intra2net AG
-License: GPLv2+
-
-
-contents
--------------------------------------------------------------------------------
-
-This module provides read and write functionality for the Intra2net *CNF*
-format. Two different syntaxes are available: classical *CNF* and a JSON
-representation. Both versions are commonly understood by Intra2net software.
-
-On the command line, raw CNF is accepted if the option ``-`` is given: ::
-
- $ get_cnf routing 2 |python3 cnfvar.py - <<ENOUGH
- 1 ROUTING,2: "192.168.55.0"
- 2 (1) ROUTING_COMMENT,0: ""
- 3 (1) ROUTING_DNS_RELAYING_ALLOWED,0: "1"
- 4 (1) ROUTING_EMAIL_RELAYING_ALLOWED,0: "1"
- 5 (1) ROUTING_FIREWALL_RULESET_REF,0: "9"
- 6 (1) ROUTING_GATEWAY,0: "10.0.254.1"
- 7 (1) ROUTING_NAT_INTO,0: "0"
- 8 (1) ROUTING_NETMASK,0: "255.255.255.0"
- 9 (1) ROUTING_PROXY_PROFILE_REF,0: "2"
- ENOUGH
-
- 1 ROUTING,2: "192.168.55.0"
- 2 (1) ROUTING_COMMENT,0: ""
- 3 (1) ROUTING_DNS_RELAYING_ALLOWED,0: "1"
- 4 (1) ROUTING_EMAIL_RELAYING_ALLOWED,0: "1"
- 5 (1) ROUTING_FIREWALL_RULESET_REF,0: "9"
- 6 (1) ROUTING_GATEWAY,0: "10.0.254.1"
- 7 (1) ROUTING_NAT_INTO,0: "0"
- 8 (1) ROUTING_NETMASK,0: "255.255.255.0"
- 9 (1) ROUTING_PROXY_PROFILE_REF,0: "2"
-
-The input takes one round-trip through the parsers and will error out on
-problematic lines. Thus, ``cnfvar.py`` can be used to syntax-check CNF data.
-
-Note that line numbers may be arbitrarily reassigned in the process. Of course,
-parent references and the relative ordering of lines will be preserved in this
-case.
-
-.. todo::
- Decide on some facility for automatic fixup of line number values. The
- internal representation is recursive so line numbers are not needed to
- establish a variable hierarchy. They might as well be omitted from the json
- input and only added when writing cnf. For the time being a lack of the
- "number" field is interpreted as an error. Though it should at least
- optionally be possible to omit the numbers entirely and have a function add
- them after the fact. This might as well be added to :py:func:`is_cnf`
- though it would be counterintuitive to have a predicate mutate its
- argument. So maybe it could return a second argument to indicate a valid
- structure that needs fixup or something like that.
-
-.. note::
- The variable values of get_cnf seems to be encoded in latin1, and set_cnf
- seems to assume latin1-encoded values (not var names). Function
- :py:func:`read_cnf` converts this to unicode and functions
- :py:func:`dump_cnf_string`, :py:func:`print_cnf`, and :py:func:`write_cnf`
- convert unicode back to latin1.
-
-
-notes on Python 3 conversion
--------------------------------------------------------------------------------
-
-Since the original *CNF* format assumes latin-1 encoded data pretty much
-exclusively, we preserve the original encoding while parsing the file.
-When assembling the data structures returned to the user, values are then
-converted to strings so they can be used naturally at the Python end.
-
-implementation
--------------------------------------------------------------------------------
-"""
-
-import functools
-import sys
-import json
-import re
-import io
-
-
-###############################################################################
-# CONSTANTS
-###############################################################################
-
-
-CNF_FIELD_MANDATORY = set ([ "varname", "data", "instance" ])
-CNF_FIELD_OPTIONAL = set ([ "parent", "children", "comment", "number" ])
-CNF_FIELD_KNOWN = CNF_FIELD_MANDATORY | CNF_FIELD_OPTIONAL
-
-grab_parent_pattern = re.compile(b"""
- ^ # match from start
- \s* # optional spaces
- \d+ # line number
- \s+ # spaces
- \((\d+)\) # parent
- """,
- re.VERBOSE)
-
-base_line_pattern = re.compile(b"""
- ^ # match from start
- \s* # optional spaces
- (\d+) # line number
- \s+ # spaces
- ([A-Z][A-Z0-9_]*) # varname
- \s* # optional spaces
- , # delimiter
- \s* # optional spaces
- (-1|\d+) # instance
- \s* # optional spaces
- : # delimiter
- \s* # optional spaces
- \"( # quoted string (data)
- (?: \\\" # (of escaped dquote
- |[^\"])* # or anything not a
- )\" # literal quote)
- \s* # optional spaces
- ( # bgroup
- \# # comment leader
- \s* # optional spaces
- .* # string (comment)
- )? # egroup, optional
- $ # eol
- """,
- re.VERBOSE)
-
-child_line_pattern = re.compile(b"""
- ^ # match from start
- \s* # optional spaces
- (\d+) # line number
- \s+ # spaces
- \((\d+)\) # parent
- \s+ # spaces
- ([A-Z][A-Z0-9_]*) # varname
- \s* # optional spaces
- , # delimiter
- \s* # optional spaces
- (-1|\d+) # instance
- \s* # optional spaces
- : # delimiter
- \s* # optional spaces
- \"([^\"]*)\" # quoted string (data)
- \s* # optional spaces
- ( # bgroup
- \# # comment leader
- \s* # optional spaces
- .* # string (comment)
- )? # egroup, optional
- $ # eol
- """,
- re.VERBOSE)
-
-
-###############################################################################
-# HELPERS
-###############################################################################
-
-
-#
-# Sadly, the Intranator is still stuck with one leg in the 90s.
-#
-def to_latin1(s):
- """Take given unicode str and convert it to a latin1-encoded `bytes`."""
- return s.encode("latin-1")
-
-
-def from_latin1(s):
- """Take given latin1-encoded `bytes` value and convert it to `str`."""
- return s.decode("latin-1")
-
-
-#
-# Conversion functions
-#
-
-def marshal_in_number(number):
- return int(number)
-
-
-def marshal_in_parent(parent):
- return int(parent)
-
-
-def marshal_in_instance(instance):
- return int(instance)
-
-
-def marshal_in_varname(varname):
- return from_latin1(varname).lower()
-
-
-def marshal_in_data(data):
- return from_latin1(data) if data is not None else ""
-
-
-def marshal_in_comment(comment):
- return comment and from_latin1(comment[1:].strip()) or None
-
-
-#
-# Type checking
-#
-
-def is_string(s):
- return isinstance(s, str)
-
-
-###############################################################################
-# EXCEPTIONS
-###############################################################################
-
-
-class InvalidCNF(Exception):
-
- def __init__(self, msg):
- self.msg = msg
-
- def __str__(self):
- return "Malformed CNF_VAR: \"%s\"" % self.msg
-
-
-class MalformedCNF(Exception):
-
- def __init__(self, msg):
- self.msg = msg
-
- def __str__(self):
- return "Malformed CNF file: \"%s\"" % self.msg
-
-
-###############################################################################
-# VALIDATION
-###############################################################################
-
-
-def is_valid(acc,
- nested,
- comment,
- data,
- instance,
- number,
- parent,
- varname):
- if varname is None:
- raise InvalidCNF("CNF_VAR lacks a name.")
- elif not is_string(varname):
- raise InvalidCNF("Varname field of CNF_VAR \"%s\" is not a string."
- % varname)
- elif varname == "":
- raise InvalidCNF("Varname field of CNF_VAR is the empty string.")
-
- if comment is not None:
- if not is_string(comment):
- raise InvalidCNF("Comment field of CNF_VAR \"%s\" is not a string."
- % varname)
-
- if data is None:
- raise InvalidCNF("Data field of CNF_VAR \"%s\" is empty."
- % varname)
- elif not is_string(data):
- raise InvalidCNF("Data field of CNF_VAR \"%s\" is not a string."
- % varname)
-
- if instance is None:
- raise InvalidCNF("Instance field of CNF_VAR \"%s\" is empty."
- % varname)
- elif not isinstance(instance, int):
- raise InvalidCNF("Instance field of CNF_VAR \"%s\" is not an integer."
- % varname)
-
- if number is None:
- raise InvalidCNF("Number field of CNF_VAR \"%s\" is empty."
- % varname)
- elif not isinstance(number, int):
- raise InvalidCNF("Number field of CNF_VAR \"%s\" is not an integer."
- % varname)
- elif number < 1:
- raise InvalidCNF("Number field of CNF_VAR \"%s\" must be positive, not %d."
- % (varname, number))
- else:
- other = acc.get(number, None)
- if other is not None: # already in use
- raise InvalidCNF("Number field of CNF_VAR \"%s\" already used by variable %s."
- % (varname, other))
- acc[number] = varname
-
- if nested is True:
- if parent is None:
- raise InvalidCNF("Parent field of nested CNF_VAR \"%s\" is empty."
- % varname)
- elif not isinstance(parent, int):
- raise InvalidCNF("Parent field of CNF_VAR \"%s\" is not an integer."
- % varname)
- else:
- if parent is not None:
- raise InvalidCNF("Flat CNF_VAR \"%s\" has nonsensical parent field \"%s\"."
- % (varname, parent))
- return acc
-
-
-def is_cnf(root):
- """
- is_cnf -- Predicate testing "CNF_VAR-ness". Folds the :py:func:`is_valid`
- predicate over the argument which can be either a well-formed CNF
- dictionary or a list of CNF_VARs.
-
- :type root: cnfvar or cnf list
- :rtype: bool
-
- Not that if it returns at all, ``is_cnf()`` returns ``True``. Any non
- well-formed member of the argument will cause the predicate to bail out
- with an exception during traversal.
- """
- cnf = cnf_root(root)
- if cnf is None:
- raise InvalidCNF(root)
- return walk_cnf(cnf, False, is_valid, {}) is not None
-
-
-def is_cnf_var(obj):
- """
- Check whether a dictionary is a valid CNF.
-
- :param dict obj: dictionary to check
- :returns: True if the dictionary has all the mandatory fields and no
- unknown fields, False otherwise
- :rtype: bool
- """
- assert isinstance (obj, dict)
-
- for f in CNF_FIELD_MANDATORY:
- if obj.get(f, None) is None:
- return False
-
- for f in obj:
- if f not in CNF_FIELD_KNOWN:
- return False
-
- return True
-
-
-###############################################################################
-# DESERIALIZATION
-###############################################################################
-
-
-#
-# JSON reader for get_cnf -j (the easy part)
-#
-
-def make_varname_lowercase(cnfvar):
- """
- Custom hook for json decoder: convert variable name to lowercase.
-
- Since variable names are case insensitive, :py:func:`read_cnf` converts
- them all to lower case. Downstream users of :py:func:`read_cnf_json` (e.g.
- :py:class:`simple_cnf.SimpleCnf`) rely on lowercase variable names.
-
- :param dict cnfvar: JSON "object" converted into a dict
- :returns: same as input but if field `varname` is present, its value is
- converted to lower case
- :rtype: dict with str keys
- """
- try:
- cnfvar['varname'] = cnfvar['varname'].lower()
- except KeyError: # there is no "varname" field
- pass
- except AttributeError: # cnfvar['varname'] is not a string
- pass
- return cnfvar
-
-
-def read_cnf_json(cnfdata):
- """
- Read json data from cnf data bytes.
-
- :param bytes cnfdata: config data
- :return: the parsed json data
- :rtype: str
-
- .. note:: The JSON module does not decode data for all versions
- of Python 3 so we handle the decoding ourselves.
- """
- if isinstance (cnfdata, bytes) is True:
- cnfdata = from_latin1 (cnfdata)
- cnf_json = json.loads(cnfdata, object_hook=make_varname_lowercase)
- if is_cnf(cnf_json) is False:
- raise TypeError("Invalid CNF_VAR.")
- return cnf_json
-
-
-#
-# CNF reader (the moderately more complicated part)
-#
-# Parsing usually starts from the `read_cnf`, which accepts a string containing
-# the variables to parse in the same structure as returned by `get_cnf`.
-#
-# In the `prepare` function the string is split into lines, and a 3-element
-# tuple is built. The first (named `current`) and second (named `next`)
-# elements of this tuple are respectively the first and second non-empty lines
-# of the input, while the third is a list of the remaining lines. This tuple is
-# named `state` in the implementation below, and it is passed around during
-# parsing. The `get` and `peek` functions are used to easily retrieve the
-# `current` and `next` items from the "state".
-#
-# When we "advance" the state, we actually drop the "current" element,
-# replacing it with the "next", while a new "next" is popped from the list of
-# remaining lines. Parsing is done this way because we need to look ahead at
-# the next line -- if it is a child it needs to be appended to the `children`
-# property of the current line.
-#
-# Regular expressions are used to extract important information from the CNF
-# lines. Finally, once parsing is completed, a dictionary is returned. The dict
-# has the same structure as the serialized JSON output returned by
-# `get_cnf -j`.
-#
-
-
-def read_cnf(data):
- """
- Read cnf data from data bytes.
-
- :param data: raw data
- :type data: str or bytes
- :return: the parsed cnf data
- :rtype: {str, {str, str or int}}
- """
- if isinstance(data, str):
- data = to_latin1(data)
- state = prepare(data)
- if state is None:
- raise InvalidCNF("Empty input string.")
-
- cnf = parse_cnf_root(state)
- if is_cnf(cnf) is False:
- raise TypeError("Invalid CNF_VAR.")
- return {"cnf": cnf}
-
-
-def prepare(raw):
- """
- Build 3-element iterable from a CNF string dump.
-
- :param raw: string content as returned by `get_cnf`
- :type raw: bytes
- :returns: 3-element tuple, where the first two elements are the first two
- lines of the output and the third is a list containing the rest
- of the lines in reverse.
- :rtype: (str * str option * str list) option
- """
- lines = raw.splitlines()
- lines.reverse()
- try:
- first = lines.pop()
- except IndexError:
- return None
-
- try:
- second = lines.pop()
- except IndexError:
- second = None
-
- first = first.strip()
- if first == b"":
- return advance((first, second, lines))
-
- return (first, second, lines)
-
-
-def advance(cns):
- """
- Pop the next line from the stream, advancing the tuple.
-
- :param cns: a 3-element tuple containing two CNF lines and a list of the
- remaining lines
- :type cnd: (str, str, [str])
- :returns: a new tuple with a new item popped from the list of lines
- :rtype cnd: (str, str, [str])
- """
- current, next, stream = cns
- if next is None: # reached end of stream
- return None
- current = next
-
- try:
- next = stream.pop()
- next = next.strip()
- except IndexError:
- next = None
-
- if current == "": # skip blank lines
- return advance((current, next, stream))
- return (current, next, stream)
-
-
-def get(cns):
- """
- Get the current line from the state without advancing it.
-
- :param cns: a 3-element tuple containing two CNF lines and a list of the
- remaining lines
- :type cnd: (str, str, [str])
- :returns: the CNF line stored as `current`
- :rtype: str
- """
- current, _, _ = cns
- return current
-
-
-def peek(cns):
- """
- Get the next line from the state without advancing it.
-
- :param cns: a 3-element tuple containing two CNF lines and a list of the
- remaining lines
- :type cnd: (str, str, [str])
- :returns: the CNF line stored as `next`
- :rtype: str
- """
- _, next, _ = cns
- return next
-
-
-def parse_cnf_root(state):
- """
- Iterate over and parse a list of CNF lines.
-
- :param state: a 3-element tuple containing two lines and a list of the
- remaining lines
- :type state: (str, str, [str])
- :returns: a list of parsed CNF variables
- :rtype: [dict]
-
- The function will parse the first element from the `state` tuple, then read
- the next line to see if it is a child variable. If it is, it will be
- appended to the last parsed CNF, otherwise top-level parsing is done
- normally.
- """
- lines = []
- current = get(state)
- while state:
- cnf_line = read_base_line(current)
- if cnf_line is not None:
- lines.append(cnf_line)
- state = advance(state)
- if state is None: # -> nothing left to do
- break
- current = get(state)
- parent = get_parent(current) # peek at next line
- if parent is not None: # -> recurse into children
- (state, children, _parent) = parse_cnf_children(state, parent)
- cnf_line["children"] = children
- if state is None:
- break
- current = get(state)
- else:
- state = advance(state)
- if state is None:
- break
- current = get(state)
- return lines
-
-
-def parse_cnf_children(state, parent):
- """
- Read and parse child CNFs of a given parent until there is none left.
-
- :param state: a 3-element tuple containing two lines and a list of the
- remaining lines
- :type state: (str, str, [str])
- :param parent: id of the parent whose children we are looking for
- :type parent: int
- :returns: a 3-element tuple with the current state, a list of children of
- the given parent and the parent ID
- :rtype: (tuple, [str], int)
-
- The function will recursively parse child lines from the `state` tuple
- until one of these conditions is satisfied:
-
- 1. the input is exhausted
- 2. the next CNF line
- 2.1. is a toplevel line
- 2.2. is a child line whose parent has a lower parent number
-
- Conceptually, 2.1 is a very similar to 2.2 but due to the special status of
- toplevel lines in CNF we need to handle them separately.
-
- Note that since nesting of CNF vars is achieved via parent line numbers,
- lines with different parents could appear out of order. libcnffile will
- happily parse those and still assign children to the specified parent:
-
- ::
- # set_cnf <<THATSALL
- 1 USER,1337: "l33t_h4x0r"
- 2 (1) USER_GROUP_MEMBER_REF,0: "2"
- 4 USER,1701: "picard"
- 5 (4) USER_GROUP_MEMBER_REF,0: "2"
- 6 (4) USER_PASSWORD,0: "engage"
- 3 (1) USER_PASSWORD,0: "hacktheplanet"
- THATSALL
- # get_cnf user 1337
- 1 USER,1337: "l33t_h4x0r"
- 2 (1) USER_GROUP_MEMBER_REF,0: "2"
- 3 (1) USER_PASSWORD,0: "hacktheplanet"
- # get_cnf user 1701
- 1 USER,1701: "picard"
- 2 (1) USER_GROUP_MEMBER_REF,0: "2"
- 3 (1) USER_PASSWORD,0: "engage"
-
- It is a limitation of ``cnfvar.py`` that it cannot parse CNF data
- structured like the above example: child lists are only populated from
- subsequent CNF vars using the parent number solely to track nesting levels.
- The parser does not keep track of line numbers while traversing the input
- so it doesn’t support retroactively assigning a child to anything else but
- the immediate parent.
- """
- lines = []
- current = get(state)
- while True:
- cnf_line = read_child_line(current)
- if cnf_line is not None:
- lines.append(cnf_line)
- state = advance(state)
- if state is None:
- break
- current = get(state)
- new_parent = get_parent(current)
- if new_parent is None:
- # drop stack
- return (state, lines, None)
- if new_parent > parent:
- # parent is further down in hierarchy -> new level
- (state, children, new_parent) = \
- parse_cnf_children (state, new_parent)
- if state is None:
- break
- cnf_line["children"] = children
- current = get(state)
- new_parent = get_parent(current)
- if new_parent is None:
- # drop stack
- return (state, lines, None)
- if new_parent < parent:
- # parent is further up in hierarchy -> pop level
- return (state, lines, new_parent)
- # new_parent == parent -> continue parsing on same level
- return (state, lines, parent)
-
-
-def get_parent(line):
- """
- Extract the ID of the parent for a given CNF line.
-
- :param str line: CNF line
- :returns: parent ID or None if no parent is found
- :rtype: int or None
- """
- match = re.match(grab_parent_pattern, line)
- if match is None: # -> no parent
- return None
- return int(match.groups()[0])
-
-
-def read_base_line(line):
- """
- Turn one top-level CNF line into a dictionary.
-
- :param str line: CNF line
- :rtype: {str: Any}
-
- This performs the necessary decoding on values to obtain proper Python
- strings from 8-bit encoded CNF data.
-
- The function only operates on individual lines. Argument strings that
- contain data for multiple lines – this includes child lines of the current
- CNF var! – will trigger a parsing exception.
- """
- if len(line.strip()) == 0:
- return None # ignore empty lines
- if line[0] == b"#":
- return None # ignore comments
-
- match = re.match(base_line_pattern, line)
- if match is None:
- raise MalformedCNF("Syntax error in line \"\"\"%s\"\"\"" % line)
- number, varname, instance, data, comment = match.groups()
- return {
- "number" : marshal_in_number (number),
- "varname" : marshal_in_varname (varname),
- "instance" : marshal_in_instance (instance),
- "data" : marshal_in_data (data),
- "comment" : marshal_in_comment (comment),
- }
-
-
-def read_child_line(line):
- """
- Turn one child CNF line into a dictionary.
-
- :param str line: CNF line
- :rtype: {str: Any}
-
- This function only operates on individual lines. If the argument string is
- syntactically valid but contains input representing multiple CNF vars, a
- parse error will be thrown.
- """
- if len(line.strip()) == 0:
- return None # ignore empty lines
- if line[0] == "#":
- return None # ignore comments
-
- match = re.match(child_line_pattern, line)
- if match is None:
- raise MalformedCNF("Syntax error in child line \"\"\"%s\"\"\""
- % from_latin1 (line))
- number, parent, varname, instance, data, comment = match.groups()
- return {
- "number" : marshal_in_number (number),
- "parent" : marshal_in_parent (parent),
- "varname" : marshal_in_varname (varname),
- "instance" : marshal_in_instance (instance),
- "data" : marshal_in_data (data),
- "comment" : marshal_in_comment (comment),
- }
-
-
-###############################################################################
-# SERIALIZATION
-###############################################################################
-
-
-cnf_line_nest_indent = " "
-cnf_line_base_fmt = "%d %s,%d: \"%s\""
-cnf_line_child_fmt = "%d %s(%d) %s,%d: \"%s\""
-
-
-def format_cnf_vars(da, var):
- """
- Return a list of formatted cnf_line byte strings.
-
- :param da: a tuple where the first element is the depth (0 = top-level,
- >1 = child CNF) and the second is the string being built.
- :type da: (int, str)
- :param var: the CNF element to convert to string in the current iteration
- :type var: dict
- :returns: a tuple like `da`, where the second element should contain all
- converted CNFs
- :rtype: (int, str)
-
- This function is meant to be passed to the :py:func:`functools.reduce`
- function.
-
- The variable names are uppercased unconditionally because while ``get_cnf``
- is case-indifferent for variable names, ``set_cnf`` isn’t.
- """
- depth, acc = da
- line = None
- if depth > 0:
- line = cnf_line_child_fmt \
- % (var["number"],
- cnf_line_nest_indent * depth,
- var["parent"],
- var["varname"].upper(),
- var["instance"],
- var["data"])
- else:
- line = cnf_line_base_fmt \
- % (var["number"],
- var["varname"].upper(),
- var["instance"],
- var["data"])
-
- comment = var.get("comment", None)
- if comment and len(comment) != 0:
- line = line + (" # %s" % comment)
-
- acc.append(to_latin1(line))
-
- children = var.get("children", None)
- if children is not None:
- (_, acc) = functools.reduce(format_cnf_vars, children, (depth + 1, acc))
-
- return (depth, acc)
-
-
-def cnf_root(root):
- """
- Extract a list of CNFs from a given structure.
-
- :param root: list of CNFs or a CNF dictionary
- :type root: [dict] or dict
- :raises: :py:class:`TypeError` if no CNFs can be extracted
- :returns: list with one or more CNF objects
- :rtype: [dict]
-
- Output varies depending on a few conditions:
- - If `root` is a list, return it right away
- - If `root` is a dict corresponding to a valid CNF value, return it wrapped
- in a list
- - If `root` is a dict with a `cnf` key containg a list (as the JSON
- returned by `get_cnf -j`), return the value
- - Otherwise, raise an error
- """
- if isinstance(root, list):
- return root
- if not isinstance(root, dict):
- raise TypeError(
- "Expected dictionary of CNF_VARs, got %s." % type(root))
- if is_cnf_var(root):
- return [root]
- cnf = root.get("cnf", None)
- if not isinstance(cnf, list):
- raise TypeError("Expected list of CNF_VARs, got %s." % type(cnf))
- return cnf
-
-
-def normalize_cnf(cnf):
- """
- Ensure the output conforms to set_cnf()’s expectations.
-
- :param cnf: list of CNF objects to normalize
- :type cnf: [dict]
- :returns: normalized list
- :rtype: [list]
- """
- if isinstance(cnf, list) is False:
- raise MalformedCNF("expected list of CNF_VARs, got [%s]" % type(cnf))
- def norm(var):
- vvar = \
- { "number" : var ["number"]
- , "varname" : var ["varname"].upper()
- , "instance" : var ["instance"]
- , "data" : var ["data"]
- }
-
- children = var.get("children", None)
- if children is not None:
- vvar ["children"] = normalize_cnf(children)
-
- parent = var.get("parent", None)
- if parent is not None:
- vvar ["parent"] = var["parent"]
-
- comment = var.get("comment", None)
- if comment is not None:
- vvar ["comment"] = var["comment"]
-
- return vvar
-
- return [norm(var) for var in cnf]
-
-
-###############################################################################
-# TRAVERSAL
-###############################################################################
-
-
-def walk_cnf(cnf, nested, fun, acc):
- """
- Depth-first traversal of a CNF tree.
-
- :type cnf: cnf list
- :type nested: bool
- :type fun: 'a -> bool -> (cnf stuff) -> 'a
- :type acc: 'a
- :rtype: 'a
-
- Executes ``fun`` recursively for each node in the tree. The function
- receives the accumulator ``acc`` which can be of an arbitrary type as first
- argument. The second argument is a flag indicating whether the current
- CNF var is a child (if ``True``) or a parent var. CNF member fields are
- passed via named optional arguments.
- """
- for var in cnf:
- acc = fun(acc,
- nested,
- comment=var.get("comment", None),
- data=var.get("data", None),
- instance=var.get("instance", None),
- number=var.get("number", None),
- parent=var.get("parent", None),
- varname=var.get("varname", None))
- children = var.get("children", None)
- if children is not None:
- acc = walk_cnf(children, True, fun, acc)
- return acc
-
-
-def renumber_vars(root, parent=None, toplevel=False):
- """
- Number cnfvars linearly.
-
- If *parent* is specified, numbering will start at this offset. Also, the
- VAR *root* will be assigned this number as a parent lineno unless
- *toplevel* is set (the root var in a CNF tree obviously can’t have a
- parent).
-
- The *toplevel* parameter is useful when renumbering an existing variable
- starting at a given offset without at the same time having that offset
- assigned as a parent.
- """
- root = cnf_root (root)
- i = parent or 0
- for var in root:
- i += 1
- var["number"] = i
- if toplevel is False and parent is not None:
- var["parent"] = parent
- children = var.get("children", None)
- if children is not None:
- i = renumber_vars(children, parent=i, toplevel=False)
- return i
-
-
-def count_vars(root):
- """
- Traverse the cnf structure recursively, counting VAR objects (CNF lines).
- """
- cnf = cnf_root(root)
- if cnf is None:
- raise InvalidCNF(root)
- return walk_cnf(cnf, True, lambda n, _nested, **_kwa: n + 1, 0)
-
-#
-# querying
-#
-
-
-def get_vars(cnf, data=None, instance=None):
- """
- get_vars -- Query a CNF_VAR structure. Skims the *toplevel* of the CNF_VAR
- structure for entries with a matching `data` or `instance` field.
-
- :type cnf: CNF_VAR
- :type data: str
- :type instance: int
- :rtype: CNF_VAR
- :returns: The structure containing only references to the
- matching variables. Containing an empty list of
- variables in case there is no match.
-
- Values are compared literally. If both ``instance`` and ``data`` are
- specified, vars will be compared against both.
- """
- cnf = cnf["cnf"]
- if cnf:
- criterion = lambda _var: False
- if data:
- if instance:
- criterion = lambda var: var[
- "data"] == data and var["instance"] == instance
- else:
- criterion = lambda var: var["data"] == data
- elif instance:
- criterion = lambda var: var["instance"] == instance
-
- return {"cnf": [var for var in cnf if criterion(var) is True]}
-
- return {"cnf": []}
-
-
-###############################################################################
-# PRINTING/DUMPING
-###############################################################################
-
-
-#
-# Print/dump raw CNF values
-#
-
-def output_cnf(root, out, renumber=False):
- """
- Dump a textual representation of given CNF VAR structure to given stream.
-
- Runs :py:func:`format_cnf_vars` on the input (`root`) and then writes that
- to the given file-like object (out).
-
- :param root: a CNF_VAR structure
- :type root: dict or list or anything that :py:func:`cnf_root` accepts
- :param out: file-like object or something with a `write(str)` function
- :param bool renumber: Whether to renumber cnfvars first
-
- Files are converted to the 8-bit format expected by CNF so they can be fed
- directly into libcnffile.
- """
- cnf = cnf_root(root)
- if renumber is True:
- _count = renumber_vars(root)
- if is_cnf(cnf) is True:
- (_, lines) = functools.reduce(format_cnf_vars, cnf, (0, []))
- if isinstance(out, (io.RawIOBase, io.BufferedIOBase)):
- out.write (b"\n".join (lines))
- out.write (b"\n")
- else: # either subclass of io.TextIOBase or unknown
- out.write ("\n".join (map (from_latin1, lines)))
- out.write ("\n")
-
-
-def dump_cnf_bytes (root, renumber=False):
- """
- Serialize CNF var structure, returning the result as a byte sequence.
- """
- cnf = cnf_root(root)
- out = io.BytesIO()
- output_cnf(root, out, renumber=renumber)
- res = out.getvalue()
- out.close()
- return res
-
-
-def dump_cnf_string(root, renumber=False):
- """
- Serialize CNF var structure, returning a latin1-encode byte string.
-
- .. todo::this is identical to :py:func:`dump_cnf_bytes`!
- """
- cnf = cnf_root(root)
- out = io.BytesIO()
- output_cnf(root, out, renumber=renumber)
- res = out.getvalue()
- out.close()
- return res
-
-
-def print_cnf(root, out=None, renumber=False):
- """
- Print given CNF_VAR structure to stdout (or other file-like object).
-
- Note that per default the config is printed to sys.stdout using the shell's
- preferred encoding. If the shell cannot handle unicode this might raise
- `UnicodeError`.
-
- All params forwarded to :py:func:`output_cnf`. See args there.
- """
- if root is not None:
- output_cnf(root, out or sys.stdout, renumber=renumber)
-
-
-def write_cnf(*argv, **kw_argv):
- """Alias for :py:func:`print_cnf`."""
- print_cnf(*argv, **kw_argv)
-
-
-def print_cnf_raw(root, out=None):
- """`if root is not None: out.write(root)`."""
- if root is not None:
- out.write(root)
-
-
-def write_cnf_raw(*argv, **kw_argv):
- """Alias for :py:func:`print_cnf_raw`."""
- print_cnf_raw(*argv, **kw_argv)
-
-
-#
-# Print/dump CNF values in JSON format
-#
-
-
-def output_json(root, out, renumber=False):
- """
- Dump CNF_VAR structure to file-like object in json format.
-
- :param root: CNF_VAR structure
- :type root: dict or list or anything that :py:func:`cnf_root` accepts
- :param out: file-like object, used as argument to :py:func:`json.dumps` so
- probably has to accept `str` (as opposed to `bytes`).
- :param bool renumber: whether to renumber variables before dupming.
- """
- # if not isinstance(out, file):
- #raise TypeError("%s (%s) is not a stream." % (out, type(out)))
- if renumber is True:
- _count = renumber_vars(root)
- if is_cnf(root) is True:
- root ["cnf"] = normalize_cnf(cnf_root (root))
- data = json.dumps(root)
- out.write(data)
- out.write("\n")
- # TODO: else raise value error?
-
-
-def dump_json_string(root, renumber=False):
- """
- Serialize CNF var structure as JSON, returning the result as a string.
- """
- out = io.StringIO()
- output_json(root, out, renumber=renumber)
- res = out.getvalue()
- out.close()
- return res
-
-
-def print_cnf_json(root, out=None, renumber=False):
- """
- Print CNF_VAR structure in json format to stdout.
-
- Calls :py:func:`output_json` with `sys.stdout` if `out` is not given or
- `None`.
- """
- if root is not None:
- output_json(root, out or sys.stdout, renumber=renumber)
-
-
-def write_cnf_json(*argv, **kw_argv):
- """Alias for :py:func:`print_cnf_json`."""
- print_cnf_json(*argv, **kw_argv)
-
-
-
-###############################################################################
-# ENTRY POINT FOR DEVELOPMENT
-###############################################################################
-
-
-def usage():
- print("usage: cnfvar.py -" , file=sys.stderr)
- print("" , file=sys.stderr)
- print(" Read CNF from stdin.", file=sys.stderr)
- print("" , file=sys.stderr)
-
-
-def main(argv):
- if len(argv) > 1:
- first = argv[1]
- if first == "-":
- cnf = read_cnf(sys.stdin.buffer.read())
- print_cnf(cnf)
- return 0
- elif first == "test":
- cnf = read_cnf(sys.stdin.buffer.read())
- cnff = get_vars(cnf, instance=2, data="FAX")
- print_cnf(cnff)
- return 0
- usage()
- return -1
-
-
-if __name__ == "__main__":
- sys.exit(main(sys.argv))
--- /dev/null
+#!/usr/bin/env python
+# This Python file uses the following encoding: utf-8
+
+# The software in this package is distributed under the GNU General
+# Public License version 2 (with a special exception described below).
+#
+# A copy of GNU General Public License (GPL) is included in this distribution,
+# in the file COPYING.GPL.
+#
+# As a special exception, if other files instantiate templates or use macros
+# or inline functions from this file, or you compile this file and link it
+# with other works to produce a work based on this file, this file
+# does not by itself cause the resulting work to be covered
+# by the GNU General Public License.
+#
+# However the source code for this file must still be made available
+# in accordance with section (3) of the GNU General Public License.
+#
+# This exception does not invalidate any other reasons why a work based
+# on this file might be covered by the GNU General Public License.
+#
+# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
+
+import os
+import unittest
+
+from src.cnfvar import string as cnfvar_old
+
+#
+# test data
+#
+
+# model cnf tree
+demo_cnfvar = {"cnf": [
+ {
+ "varname": "MY_FAVORITE_CNF_VAR",
+ "instance": 1337,
+ "number": 42,
+ "data": "string conf content",
+ "comment": ""
+ },
+ {
+ "varname": "SOME_NESTED_CNF_VAR",
+ "instance": 0,
+ "number": 23,
+ "data": "999",
+ "comment": "",
+ "children": [
+ {
+ "varname": "SOME_CHILD_VAR",
+ "instance": 1,
+ "number": 24,
+ "data": "2014",
+ "parent": 23,
+ "comment": ""
+ }
+ ]
+ },
+]}
+
+# duplicate line number
+demo_invalid_cnfvar = {"cnf": [
+ {
+ "varname": "SOME_PARTICULARLY_TASTY_CNF_VAR",
+ "instance": 1337,
+ "number": 23,
+ "data": "classic wingers",
+ "comment": ""
+ },
+ {
+ "varname": "EXTRAORDINARILY_FANCY_CNF_VAR",
+ "instance": 1,
+ "number": 42,
+ "data": "ab mentions",
+ "comment": ""
+ },
+ {
+ "varname": "ANOTHER_POPULAR_CNF_VAR",
+ "instance": 0,
+ "number": 42,
+ "data": "notches",
+ "comment": ""
+ }
+]}
+
+demo_nonascii = r"""
+1 USER,1: "admin"
+2 (1) USER_DISABLED,0: "0"
+3 (1) USER_FULLNAME,0: "Administrator"
+4 (1) USER_GROUPWARE_FOLDER_CALENDAR,0: "INBOX/Kalender"
+5 (1) USER_GROUPWARE_FOLDER_CONTACTS,0: "INBOX/Kontakte"
+6 (1) USER_GROUPWARE_FOLDER_DRAFTS,0: "INBOX/Entwürfe"
+7 (1) USER_GROUPWARE_FOLDER_NOTES,0: "INBOX/Notizen"
+8 (1) USER_GROUPWARE_FOLDER_OUTBOX,0: "INBOX/Gesendete Elemente"
+9 (1) USER_GROUPWARE_FOLDER_TASKS,0: "INBOX/Aufgaben"
+10 (1) USER_GROUPWARE_FOLDER_TRASH,0: "INBOX/Gelöschte Elemente"
+11 (1) USER_GROUP_MEMBER_REF,0: "1"
+12 (1) USER_GROUP_MEMBER_REF,1: "2"
+13 (1) USER_PASSWORD,0: "test1234"
+"""
+
+demo_latin1crap = demo_nonascii.encode('latin1')
+
+demo_cnf_group = """
+1 GROUP,1: "Administratoren"
+2 (1) GROUP_ACCESS_GO_ONLINE_ALLOWED,0: "1"
+3 (1) GROUP_EMAILFILTER_BAN_FILTERLIST_REF,0: "-1"
+4 (1) GROUP_EMAIL_RELAY_RIGHTS,0: "RELAY_FROM_INTRANET"
+5 (1) GROUP_PROXY_PROFILE_REF,0: "1"
+"""
+
+demo_cnf_group_bytes = demo_cnf_group.encode("latin-1")
+
+demo_cnf_filter = b"""
+1 EMAILFILTER_BAN_FILTERLIST,1: "Vordefiniert: Alles verboten"
+2 (1) EMAILFILTER_BAN_FILTERLIST_ENCRYPTED,0: "BLOCK"
+3 (1) EMAILFILTER_BAN_FILTERLIST_EXTENSIONS,0: ""
+4 (1) EMAILFILTER_BAN_FILTERLIST_MIMETYPES,0: ""
+5 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,0: "text/plain"
+6 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,1: "text/html"
+7 (1) EMAILFILTER_BAN_FILTERLIST_MODE,0: "ALLOW"
+8 (1) EMAILFILTER_BAN_FILTERLIST_PREDEFINED_ID,0: "1"
+"""
+
+demo_cnf_comments = b"""
+1 EMAILFILTER_BAN_FILTERLIST,1: "Vordefiniert: Alles verboten"
+2 (1) EMAILFILTER_BAN_FILTERLIST_ENCRYPTED,0: "BLOCK"
+3 (1) EMAILFILTER_BAN_FILTERLIST_EXTENSIONS,0: ""
+4 (1) EMAILFILTER_BAN_FILTERLIST_MIMETYPES,0: "" # foo
+5 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,0: "text/plain"#bar
+6 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,1: "text/html"
+7 (1) EMAILFILTER_BAN_FILTERLIST_MODE,0: "ALLOW" # baz
+8 (1) EMAILFILTER_BAN_FILTERLIST_PREDEFINED_ID,0: "1"
+"""
+
+demo_cnf_escaped_quotes = """
+1 HERE_BE_QUOTES,0: "\""
+2 HERE_BE_QUOTES,1: "foo\"bar\"\"\"baz"
+3 HERE_BE_QUOTES,2: "unquo\\\"table"
+4 HERE_BE_QUOTES,3: "unquo\\\\\"\"table"
+"""
+
+#
+# test class
+#
+
+class CnfVarUnittest(unittest.TestCase):
+
+ def test_print_cnf(self):
+ with open(os.devnull, "w") as devnull:
+ print(demo_cnfvar, file=devnull)
+
+ def test_parse_cnf_simple_str(self):
+ cnf = cnfvar_old.read_cnf(demo_cnf_group)
+ with open(os.devnull, "w") as devnull:
+ print(cnf, file=devnull)
+
+ def test_parse_cnf_simple_bytes(self):
+ cnf = cnfvar_old.read_cnf(demo_cnf_group_bytes)
+ with open(os.devnull, "w") as devnull:
+ print(cnf, file=devnull)
+
+ def test_parse_cnf_nested(self):
+ cnf = cnfvar_old.read_cnf(demo_cnf_filter)
+ with open(os.devnull, "w") as devnull:
+ print(cnf, file=devnull)
+
+ def test_parse_cnf_comments(self):
+ cnf = cnfvar_old.read_cnf(demo_cnf_comments)
+ with open(os.devnull, "w") as devnull:
+ print(cnf, file=devnull)
+
+ def test_print_cnf_garbage(self):
+ try:
+ with open(os.devnull, "w") as devnull:
+ print(demo_invalid_cnfvar, file=devnull)
+ except cnfvar_old.InvalidCNF:
+ print ("Caught the duplicate line, bravo!")
+
+ def test_parse_cnf_quotes(self):
+ cnf = cnfvar_old.read_cnf(demo_cnf_escaped_quotes)
+ with open(os.devnull, "w") as devnull:
+ print(demo_invalid_cnfvar, file=devnull)
+
+ def test_read_nonascii(self):
+ cnf = cnfvar_old.read_cnf(demo_nonascii)
+ with open(os.devnull, "w") as devnull:
+ print(cnf, file=devnull)
+
+ def test_read_latin1(self):
+ cnf = cnfvar_old.read_cnf(demo_latin1crap)
+ with open(os.devnull, "w") as devnull:
+ print(cnf, file=devnull)
+
+
+class CnfVarUnittestVarnameCase(unittest.TestCase):
+ """Tests for verifying that uppercasing/lowercasing of varname works."""
+ # TODO: rethink whether this lower-casing is worth all the effort it causes
+
+ def test_read_cnf_lowercase(self):
+ """Test that after reading, varnames are lowercase."""
+ cnf = cnfvar_old.read_cnf(demo_cnf_group_bytes)
+ for parentvar in cnf['cnf']:
+ self.assertEqual(parentvar['varname'],
+ parentvar['varname'].lower())
+ if 'children' in parentvar:
+ for childvar in parentvar['children']:
+ self.assertEqual(parentvar['varname'],
+ parentvar['varname'].lower())
+
+
+if __name__ == '__main__':
+ unittest.main()
+++ /dev/null
-#!/usr/bin/env python
-# This Python file uses the following encoding: utf-8
-
-# The software in this package is distributed under the GNU General
-# Public License version 2 (with a special exception described below).
-#
-# A copy of GNU General Public License (GPL) is included in this distribution,
-# in the file COPYING.GPL.
-#
-# As a special exception, if other files instantiate templates or use macros
-# or inline functions from this file, or you compile this file and link it
-# with other works to produce a work based on this file, this file
-# does not by itself cause the resulting work to be covered
-# by the GNU General Public License.
-#
-# However the source code for this file must still be made available
-# in accordance with section (3) of the GNU General Public License.
-#
-# This exception does not invalidate any other reasons why a work based
-# on this file might be covered by the GNU General Public License.
-#
-# Copyright (c) 2016-2018 Intra2net AG <info@intra2net.com>
-
-import os
-import unittest
-
-from src import cnfvar_old
-
-#
-# test data
-#
-
-# model cnf tree
-demo_cnfvar = {"cnf": [
- {
- "varname": "MY_FAVORITE_CNF_VAR",
- "instance": 1337,
- "number": 42,
- "data": "string conf content",
- "comment": ""
- },
- {
- "varname": "SOME_NESTED_CNF_VAR",
- "instance": 0,
- "number": 23,
- "data": "999",
- "comment": "",
- "children": [
- {
- "varname": "SOME_CHILD_VAR",
- "instance": 1,
- "number": 24,
- "data": "2014",
- "parent": 23,
- "comment": ""
- }
- ]
- },
-]}
-
-# duplicate line number
-demo_invalid_cnfvar = {"cnf": [
- {
- "varname": "SOME_PARTICULARLY_TASTY_CNF_VAR",
- "instance": 1337,
- "number": 23,
- "data": "classic wingers",
- "comment": ""
- },
- {
- "varname": "EXTRAORDINARILY_FANCY_CNF_VAR",
- "instance": 1,
- "number": 42,
- "data": "ab mentions",
- "comment": ""
- },
- {
- "varname": "ANOTHER_POPULAR_CNF_VAR",
- "instance": 0,
- "number": 42,
- "data": "notches",
- "comment": ""
- }
-]}
-
-demo_jsoncnf = """
-{
- "cnf" : [
- {
- "children" : [
- {
- "comment" : "",
- "data" : "1",
- "instance" : 0,
- "number" : 2,
- "parent" : 1,
- "varname" : "FIREWALL_SERVICEGROUP_PREDEFINED_ID"
- },
- {
- "children" : [
- {
- "comment" : "",
- "data" : "",
- "instance" : 0,
- "number" : 4,
- "parent" : 3,
- "varname" : "FIREWALL_SERVICEGROUP_TYPE_COMMENT"
- },
- {
- "comment" : "",
- "data" : "80",
- "instance" : 0,
- "number" : 5,
- "parent" : 3,
- "varname" : "FIREWALL_SERVICEGROUP_TYPE_TCPUDP_DST_PORT"
- }
- ],
- "comment" : "",
- "data" : "TCP",
- "instance" : 0,
- "number" : 3,
- "parent" : 1,
- "varname" : "FIREWALL_SERVICEGROUP_TYPE"
- }
- ],
- "comment" : "",
- "data" : "http",
- "instance" : 1,
- "number" : 1,
- "varname" : "FIREWALL_SERVICEGROUP"
- },
- {
- "children" : [
- {
- "comment" : "",
- "data" : "2",
- "instance" : 0,
- "number" : 7,
- "parent" : 6,
- "varname" : "FIREWALL_SERVICEGROUP_PREDEFINED_ID"
- },
- {
- "children" : [
- {
- "comment" : "",
- "data" : "",
- "instance" : 0,
- "number" : 9,
- "parent" : 8,
- "varname" : "FIREWALL_SERVICEGROUP_TYPE_COMMENT"
- },
- {
- "comment" : "",
- "data" : "443",
- "instance" : 0,
- "number" : 10,
- "parent" : 8,
- "varname" : "FIREWALL_SERVICEGROUP_TYPE_TCPUDP_DST_PORT"
- }
- ],
- "comment" : "",
- "data" : "TCP",
- "instance" : 0,
- "number" : 8,
- "parent" : 6,
- "varname" : "FIREWALL_SERVICEGROUP_TYPE"
- }
- ],
- "comment" : "",
- "data" : "https",
- "instance" : 2,
- "number" : 6,
- "varname" : "FIREWALL_SERVICEGROUP"
- }
- ]
-}
-"""
-
-demo_jsoncnf_bytes = demo_jsoncnf.encode ("latin-1")
-
-demo_nonascii = r"""
-{ "cnf" : [
- {
- "children" : [
- {
- "comment" : "",
- "data" : "0",
- "instance" : 0,
- "number" : 2,
- "parent" : 1,
- "varname" : "USER_DISABLED"
- },
- {
- "comment" : "",
- "data" : "Administrator",
- "instance" : 0,
- "number" : 3,
- "parent" : 1,
- "varname" : "USER_FULLNAME"
- },
- {
- "comment" : "",
- "data" : "INBOX/Kalender",
- "instance" : 0,
- "number" : 4,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_CALENDAR"
- },
- {
- "comment" : "",
- "data" : "INBOX/Kontakte",
- "instance" : 0,
- "number" : 5,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_CONTACTS"
- },
- {
- "comment" : "",
- "data" : "INBOX/Entwürfe",
- "instance" : 0,
- "number" : 6,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_DRAFTS"
- },
- {
- "comment" : "",
- "data" : "INBOX/Notizen",
- "instance" : 0,
- "number" : 7,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_NOTES"
- },
- {
- "comment" : "",
- "data" : "INBOX/Gesendete Elemente",
- "instance" : 0,
- "number" : 8,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_OUTBOX"
- },
- {
- "comment" : "",
- "data" : "INBOX/Aufgaben",
- "instance" : 0,
- "number" : 9,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_TASKS"
- },
-
- {
- "comment" : "",
- "data" : "INBOX/Gelöschte Elemente",
- "instance" : 0,
- "number" : 10,
- "parent" : 1,
- "varname" : "USER_GROUPWARE_FOLDER_TRASH"
- },
- {
- "comment" : "",
- "data" : "1",
- "instance" : 0,
- "number" : 11,
- "parent" : 1,
- "varname" : "USER_GROUP_MEMBER_REF"
- },
- {
- "comment" : "",
- "data" : "2",
- "instance" : 1,
- "number" : 12,
- "parent" : 1,
- "varname" : "USER_GROUP_MEMBER_REF"
- },
- {
- "comment" : "",
- "data" : "",
- "instance" : 0,
- "number" : 13,
- "parent" : 1,
- "varname" : "USER_LOCALE"
- },
- {
- "comment" : "",
- "data" : "idkfa",
- "instance" : 0,
- "number" : 14,
- "parent" : 1,
- "varname" : "USER_PASSWORD"
- },
- {
- "comment" : "",
- "data" : "30",
- "instance" : 0,
- "number" : 15,
- "parent" : 1,
- "varname" : "USER_TRASH_DELETEDAYS"
- }
- ],
- "comment" : "",
- "data" : "admin",
- "instance" : 1,
- "number" : 1,
- "varname" : "USER"
- }
-]}
-"""
-
-demo_latin1crap = demo_nonascii.encode('latin1')
-
-demo_cnf_group = """
-1 GROUP,1: "Administratoren"
-2 (1) GROUP_ACCESS_GO_ONLINE_ALLOWED,0: "1"
-3 (1) GROUP_EMAILFILTER_BAN_FILTERLIST_REF,0: "-1"
-4 (1) GROUP_EMAIL_RELAY_RIGHTS,0: "RELAY_FROM_INTRANET"
-5 (1) GROUP_PROXY_PROFILE_REF,0: "1"
-"""
-
-demo_cnf_group_bytes = demo_cnf_group.encode ("latin-1")
-
-demo_cnf_filter = b"""
-1 EMAILFILTER_BAN_FILTERLIST,1: "Vordefiniert: Alles verboten"
-2 (1) EMAILFILTER_BAN_FILTERLIST_ENCRYPTED,0: "BLOCK"
-3 (1) EMAILFILTER_BAN_FILTERLIST_EXTENSIONS,0: ""
-4 (1) EMAILFILTER_BAN_FILTERLIST_MIMETYPES,0: ""
-5 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,0: "text/plain"
-6 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,1: "text/html"
-7 (1) EMAILFILTER_BAN_FILTERLIST_MODE,0: "ALLOW"
-8 (1) EMAILFILTER_BAN_FILTERLIST_PREDEFINED_ID,0: "1"
-"""
-
-demo_cnf_comments = b"""
-1 EMAILFILTER_BAN_FILTERLIST,1: "Vordefiniert: Alles verboten"
-2 (1) EMAILFILTER_BAN_FILTERLIST_ENCRYPTED,0: "BLOCK"
-3 (1) EMAILFILTER_BAN_FILTERLIST_EXTENSIONS,0: ""
-4 (1) EMAILFILTER_BAN_FILTERLIST_MIMETYPES,0: "" # foo
-5 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,0: "text/plain"#bar
-6 (4) EMAILFILTER_BAN_FILTERLIST_MIMETYPES_NAME,1: "text/html"
-7 (1) EMAILFILTER_BAN_FILTERLIST_MODE,0: "ALLOW" # baz
-8 (1) EMAILFILTER_BAN_FILTERLIST_PREDEFINED_ID,0: "1"
-"""
-
-demo_cnf_escaped_quotes = """
-1 HERE_BE_QUOTES,0: "\""
-2 HERE_BE_QUOTES,1: "foo\"bar\"\"\"baz"
-3 HERE_BE_QUOTES,2: "unquo\\\"table"
-4 HERE_BE_QUOTES,3: "unquo\\\\\"\"table"
-"""
-
-demo_json_escaped_quotes = """
-{ "cnf": [ { "number" : 1,
- "varname" : "HERE_BE_QUOTES",
- "instance" : 0,
- "data" : "\\"" },
- { "number" : 2,
- "varname" : "HERE_BE_QUOTES",
- "instance" : 1,
- "data" : "foo\\"bar\\"\\"\\"baz" },
- { "number" : 3,
- "varname" : "HERE_BE_QUOTES",
- "instance" : 2,
- "data" : "unquo\\\\\\"table" },
- { "number" : 4,
- "varname" : "HERE_BE_QUOTES",
- "instance" : 3,
- "data" : "unquo\\\\\\\\\\"\\"table" } ] }
-"""
-
-#
-# test class
-#
-
-class CnfVarUnittest(unittest.TestCase):
-
- def test_print_cnf(self):
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf(demo_cnfvar, out=devnull)
-
- def test_parse_cnf_simple_str(self):
- cnf = cnfvar_old.read_cnf(demo_cnf_group)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf_json(cnf, out=devnull)
-
- def test_parse_cnf_simple_bytes(self):
- cnf = cnfvar_old.read_cnf(demo_cnf_group_bytes)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf_json(cnf, out=devnull)
-
- def test_parse_cnf_nested(self):
- cnf = cnfvar_old.read_cnf(demo_cnf_filter)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf_json(cnf, out=devnull)
-
- def test_parse_cnf_comments(self):
- cnf = cnfvar_old.read_cnf(demo_cnf_comments)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf_json(cnf, out=devnull)
-
- def test_print_cnf_garbage(self):
- try:
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf(demo_invalid_cnfvar, out=devnull)
- except cnfvar_old.InvalidCNF:
- print ("Caught the duplicate line, bravo!")
-
- def test_read_json_str(self):
- cnf = cnfvar_old.read_cnf_json(demo_jsoncnf)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf(cnf, out=devnull)
-
- def test_read_json_bytes(self):
- cnf = cnfvar_old.read_cnf_json(demo_jsoncnf_bytes)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf(cnf, out=devnull)
-
- def test_read_json_nonascii(self):
- cnf = cnfvar_old.read_cnf_json(demo_nonascii)
- with open(os.devnull, "wb") as devnull:
- cnfvar_old.print_cnf(cnf, out=devnull)
-
- def test_read_json_latin1(self):
- cnf = cnfvar_old.read_cnf_json(demo_latin1crap)
- with open(os.devnull, "wb") as devnull:
- cnfvar_old.print_cnf(cnf, out=devnull)
-
- def test_parse_cnf_quotes(self):
- cnf = cnfvar_old.read_cnf(demo_cnf_escaped_quotes)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf_json(cnf, out=devnull)
-
- def test_parse_json_quotes(self):
- cnf = cnfvar_old.read_cnf_json(demo_json_escaped_quotes)
- with open(os.devnull, "w") as devnull:
- cnfvar_old.print_cnf_json(cnf, out=devnull)
-
-
-class CnfVarUnittestVarnameCase(unittest.TestCase):
- """Tests for verifying that uppercasing/lowercasing of varname works."""
- # TODO: rethink whether this lower-casing is worth all the effort it causes
-
- def test_dump_cnf_uppercase(self):
- """Test dump of cnfvars results in uppercase var names."""
- cnf = {"cnf": [
- {"instance": 0, "varname": "dialout_mode",
- "data": "ONLINE", "number": 1, "comment": None},
- {"instance": 0, "varname": "dialout_defaultprovider_ref",
- "data": "1", "number": 2, "comment": None},
- {"instance": 0, "varname": "hypothetical_parent",
- "data": "parent value", "number": 3, "comment": None,
- "children": [
- {"instance": 0, "varname": "hypothetical_child",
- "data": "0", "number": 4, "parent": 3, "comment": None},
- {"instance": 1, "varname": "hypothetical_child",
- "data": "1", "number": 5, "parent": 3, "comment": None}]}
- ]}
- serialization = cnfvar_old.dump_json_string(cnf)
- self.assertIn('DIALOUT_MODE', serialization)
- self.assertIn('DIALOUT_DEFAULTPROVIDER_REF', serialization)
- self.assertIn('HYPOTHETICAL_CHILD', serialization)
- self.assertNotIn('dialout_mode', serialization)
- self.assertNotIn('dialout_defaultprovider_ref', serialization)
- self.assertNotIn('hypothetical_child', serialization)
-
- def test_read_cnf_lowercase(self):
- """Test that after reading, varnames are lowercase."""
- cnf = cnfvar_old.read_cnf_json(demo_jsoncnf.encode('latin1'))
- for parentvar in cnf['cnf']:
- self.assertEqual(parentvar['varname'],
- parentvar['varname'].lower())
- if 'children' in parentvar:
- for childvar in parentvar['children']:
- self.assertEqual(parentvar['varname'],
- parentvar['varname'].lower())
-
-
-if __name__ == '__main__':
- unittest.main()
git://developer.intra2net.com / pyi2ncommon / commitdiff