- ``InvalidGCMTag`` (decryption failed on account of an invalid GCM
tag),
- ``InvalidIVFixedPart`` (IV fixed part of object not found in list),
- - ``DuplicateIV`` (the IV of an encrypted object already occurred),
+ - ``DuplicateIV`` (the IV of an object encrypted earlier was reused),
+ - ``NonConsecutiveIV`` (IVs of two encrypted objects are not
+ consecutive),
- ``DecryptionError`` (used in CLI decryption for presenting error
conditions to the user).
has no notion of a position in a PDT encrypted archive, this condition must be
sorted out downstream.
+When encrypting with more than one Encrypt context special care must be taken
+to prevent accidental reuse of IVs. The builtin protection against reuse is
+only effective for objects encrypted with the same Encrypt handle. If multiple
+Encrypt handles are used to encrypt with the same combination of password and
+salt, the encryption becomes susceptible to birthday attacks (bound = 2^32 due
+to the 64-bit random iv). Thus the use of multiple handles is discouraged.
+
+
Command Line Utility
-------------------------------------------------------------------------------
other parameters may change. Thus a key derived using above method from the
first object doesn’t necessarily apply to any of the subsequent objects.
+Future Developments
+-------------------------------------------------------------------------------
+
+As of 2020 with the format version 1, Deltatar encryption uses the AES-GCM mode
+which requires meticulous bookkeeping of initialization vectors. A future
+version could simplify this aspect of the encryption by switching to the more
+recent AES-GCM-SIV mode (RFC 8452).
"""
import base64
, "NaCl_LEN" : 16 })
, "enc": "aes-gcm" } }
+# Mode zero is unencrypted and only provided for testing purposes. nless
+# the encryptor / decryptor are explicitly instructed to do so.
+MIN_SECURE_PARAMETERS = 1
+
###############################################################################
## constants
###############################################################################
def iv_fmt (iv):
"""Format the two components of an IV in a readable fashion."""
fixed, cnt = struct.unpack (FMT_I2N_IV, iv)
- return IV_FMT % (binascii.hexlify (fixed), cnt)
+ return IV_FMT % (binascii.hexlify (fixed).decode (), cnt)
###############################################################################
(kdf, params) = defs["kdf"]
fn = None
if kdf == "scrypt" : fn = kdf_scrypt
- if kdf == "dummy" : fn = kdf_dummy
+ elif kdf == "dummy" : fn = kdf_dummy
if fn is None:
raise ValueError ("key derivation method %r unknown" % kdf)
return partial (fn, params)
iv = None # current IV
fixed = None # accu for 64 bit fixed parts of IV
used_ivs = None # tracks IVs
- strict_ivs = False # if True, panic on duplicate object IV
+ strict_ivs = False # if True, panic on duplicate or non-consecutive object IV
password = None
paramversion = None
+ insecure = False # allow plaintext parameters
stats = { "in" : 0
, "out" : 0
, "obj" : 0 }
elif cnt == AES_GCM_IV_CNT_INDEX:
if self.index_counter_used is True:
raise InvalidFileCounter ("attempted to reuse index file "
- " counter %d: must be unique" % cnt)
+ "counter %d: must be unique" % cnt)
self.index_counter_used = True
if cnt <= AES_GCM_IV_CNT_MAX:
self.cnt = cnt
def set_parameters (self, password=None, key=None, paramversion=None,
- nacl=None, counter=None, strict_ivs=False):
+ nacl=None, counter=None, strict_ivs=False,
+ insecure=False):
"""
Configure the internal state of a crypto context. Not intended for
external use.
+
+ A parameter version indicating passthrough (plaintext) mode is rejected
+ with an ``InvalidParameter`` unless ``insecure`` is set.
"""
self.next_fixed ()
self.set_object_counter (counter)
self.strict_ivs = strict_ivs
+ self.insecure = insecure
+
if paramversion is not None:
+ if self.insecure is False \
+ and paramversion < MIN_SECURE_PARAMETERS:
+ raise InvalidParameter \
+ ("set_parameters: requested parameter version %d but "
+ "plaintext encryption disallowed in secure context!"
+ % paramversion)
self.paramversion = paramversion
if key is not None:
return out
- def next (self, password, paramversion, nacl, iv):
+ def next (self, password, paramversion, nacl):
"""
Prepare for encrypting another object: Reset the data counters and
change the configuration in case one of the variable parameters differs
- from the last object. Also check the IV for duplicates and error out
- if strict checking was requested.
+ from the last object.
"""
self.ctsize = 0
self.ptsize = 0
self.stats ["obj"] += 1
- self.check_duplicate_iv (iv)
-
if ( self.paramversion != paramversion
or self.password != password
or self.nacl != nacl):
self.set_parameters (password=password, paramversion=paramversion,
- nacl=nacl, strict_ivs=self.strict_ivs)
-
-
- def check_duplicate_iv (self, iv):
- """
- Add an IV (the 12 byte representation as in the header) to the list. With
- strict checking enabled, this will throw a ``DuplicateIV``. Depending on
- the context, this may indicate a serious error (IV reuse).
- """
- if self.strict_ivs is True and iv in self.used_ivs:
- raise DuplicateIV ("iv %s was reused" % iv_fmt (iv))
- # vi has not been used before; add to collection
- self.used_ivs.add (iv)
+ nacl=nacl, strict_ivs=self.strict_ivs,
+ insecure=self.insecure)
def counters (self):
self.enc = None
+ def get_used_ivs (self):
+ """
+ Get the set of IVs that were used so far during the lifetime of
+ this context. Useful to check for IV reuse if multiple encryption
+ contexts were used independently.
+ """
+ return self.used_ivs
+
+
+ def reset_last_iv (self):
+ """
+ Implemented only for decryptor; no-op otherwise.
+ """
+ pass
+
+
class Encrypt (Crypto):
lastinfo = None
paramenc = None
def __init__ (self, version, paramversion, password=None, key=None, nacl=None,
- counter=AES_GCM_IV_CNT_DATA, strict_ivs=True):
+ counter=AES_GCM_IV_CNT_DATA, strict_ivs=False, insecure=False):
"""
The ctor will throw immediately if one of the parameters does not conform
to our expectations.
- counter=AES_GCM_IV_CNT_DATA, strict_ivs=True):
:type version: int to fit uint16_t
:type paramversion: int to fit uint16_t
:param password: mutually exclusive with ``key``
``AES_GCM_IV_CNT_INDEX`` are unique in each backup set
and cannot be reused even with different fixed parts.
:type strict_ivs: bool
+ :param strict_ivs: Enable paranoid tracking of IVs.
+ :type insecure: bool
+ :param insecure: whether to permit passthrough mode
+
+ *Security considerations*: The ``class Encrypt`` handle guarantees that
+ all random parts (first eight bytes) of the IVs used for encrypting
+ objects are unique. This guarantee does *not* apply across handles if
+ multiple handles are used with the same combination of password and
+ salt. Thus, use of multiple handles with the same combination of password
+ and salt is subject to birthday attacks with a bound of 2^32. To avoid
+ collisions, the application should keep the number of handles as low
+ as possible and check for reuse by comparing the set of IVs used of all
+ handles that were created (accessible using the ``get_used_ivs`` method).
"""
if password is None and key is None \
or password is not None and key is not None :
self.paramenc = ENCRYPTION_PARAMETERS.get (paramversion) ["enc"]
super().__init__ (password, key, paramversion, nacl, counter=counter,
- strict_ivs=strict_ivs)
+ strict_ivs=strict_ivs, insecure=insecure)
def next_fixed (self, retries=PDTCRYPT_IV_GEN_MAX_RETRIES):
% self.paramversion)
hdrdum = hdr_make_dummy (filename)
self.lastinfo = (filename, hdrdum)
- super().next (self.password, self.paramversion, self.nacl, self.iv)
+
+ self.check_duplicate_iv (self.iv)
+
+ super().next (self.password, self.paramversion, self.nacl)
self.set_object_counter (self.cnt + 1)
return hdrdum
+ def check_duplicate_iv (self, iv):
+ """
+ Add an IV (the 12 byte representation as in the header) to the list. With
+ strict checking enabled, this will throw a ``DuplicateIV``. Depending on
+ the context, this may indicate a serious error (IV reuse).
+
+ IVs are only tracked in strict_ivs mode.
+ """
+ if self.strict_ivs is False:
+ return
+
+ if iv in self.used_ivs:
+ raise DuplicateIV ("iv %s was reused" % iv_fmt (iv))
+ # vi has not been used before; add to collection
+ self.used_ivs.add (iv)
+
+
def done (self, cmpdata):
"""
Complete encryption of an object. After this has been called, attempts
hdr_ctsize = -1
def __init__ (self, password=None, key=None, counter=None, fixedparts=None,
- strict_ivs=False):
+ strict_ivs=True, insecure=False):
"""
Sanitizing ctor for the decryption context. ``fixedparts`` specifies a
list of IV fixed parts accepted during decryption. If a fixed part is
``AES_GCM_IV_CNT_INDEX`` are unique in each backup set
and cannot be reused even with different fixed parts.
:type fixedparts: bytes list
+ :type strict_ivs: bool
+ :param strict_ivs: fail if IVs of decrypted objects are not linearly
+ increasing
+ :type insecure: bool
+ :param insecure: whether to process objects encrypted in
+ passthrough mode (*``paramversion`` < 1*)
+
+ *Security considerations*: The ``strict_ivs`` setting protects against
+ ciphertext reordering and injection attacks. For this to work it relies
+ on a property of how the object counters are created during encryption.
+ If multiple ``Encrypt`` handles have been used during encryption, this
+ is property is unlikely to apply as it would require manual management
+ of counters across Encrypt handles. In these cases it may thus be
+ necessary to disable the ```strict_ivs`` protection.
"""
if password is None and key is None \
or password is not None and key is not None :
self.fixed.sort ()
super().__init__ (password=password, key=key, counter=counter,
- strict_ivs=strict_ivs)
+ strict_ivs=strict_ivs, insecure=insecure)
def valid_fixed_part (self, iv):
return i != len (self.fixed) and self.fixed [i] == fixed
+ def reset_last_iv (self):
+ """
+ Force a new IV sequence start. The last IV counter will be set from the
+ next IV encountered and the check for consecutive IVs will be suppressed.
+
+ The intended use is backup volume boundaries or handling batches of
+ objects encrypted with ``Encrypt`` handles initialized with different
+ initial counter values.
+ """
+ self.last_iv = None
+
def check_consecutive_iv (self, iv):
"""
Check whether the counter part of the given IV is indeed the successor
if self.strict_ivs is True \
and self.last_iv is not None \
and self.last_iv [0] == fixed \
- and self.last_iv [1] != cnt - 1:
+ and self.last_iv [1] + 1 != cnt:
raise NonConsecutiveIV ("iv %s counter not successor of "
"last object (expected %d, found %d)"
- % (iv_fmt (iv), self.last_iv [1], cnt))
+ % (iv_fmt (iv), self.last_iv [1] + 1, cnt))
self.last_iv = (fixed, cnt)
self.hdr_ctsize = ctsize
- super().next (self.password, paramversion, nacl, iv)
+ super().next (self.password, paramversion, nacl)
if self.fixed is not None and self.valid_fixed_part (iv) is False:
raise InvalidIVFixedPart ("iv %s has invalid fixed part"
% iv_fmt (iv))
+
self.check_consecutive_iv (iv)
self.tag = tag
PDTCRYPT_SUB_PROCESS = 0
PDTCRYPT_SUB_SCRYPT = 1
PDTCRYPT_SUB_SCAN = 2
+PDTCRYPT_SUB_IVCHECK = 3
PDTCRYPT_SUB = \
{ "process" : PDTCRYPT_SUB_PROCESS
, "scrypt" : PDTCRYPT_SUB_SCRYPT
- , "scan" : PDTCRYPT_SUB_SCAN }
+ , "scan" : PDTCRYPT_SUB_SCAN
+ , "ivcheck" : PDTCRYPT_SUB_IVCHECK }
PDTCRYPT_DECRYPT = 1 << 0 # decrypt archive with password
PDTCRYPT_SPLIT = 1 << 1 # split archive into individual objects
return d
+def check_ivs (ifs):
+ """
+ Walk the objects in the given reader, validating uniqueness and
+ consecutiveness of the IVs in the object headers.
+
+ As the IVs are metadata this does not require decryption.
+ """
+ objs = 0
+ seen = set ()
+ last = None
+
+ while True:
+ try:
+ hdr = hdr_read_stream (ifs)
+ except EndOfFile as exn:
+ break # done
+
+ objs += 1
+ cur = hdr ["iv"]
+
+ fixed, cnt = struct.unpack (FMT_I2N_IV, cur)
+
+ if PDTCRYPT_VERBOSE is True:
+ noise ("PDT: obj %d, iv %s" % (objs, iv_fmt (cur)))
+
+ if last is not None:
+ if fixed != last [0]:
+ noise ("PDT: obj %d, fixed part changed last: %s → this: %s"
+ % (obj,
+ binascii.hexlify (last [0]),
+ binascii.hexlify (fixed)))
+ if cnt != last [1] + 1:
+ raise NonConsecutiveIV ("iv %s counter not successor of "
+ "last object (expected %d, found %d)"
+ % (iv_fmt (cur), last [1] + 1, cnt))
+
+ if cur in seen:
+ raise DuplicateIV ("iv %s was reused" % iv_fmt (cur))
+
+ seen.add (cur)
+ last = (fixed, cnt)
+
+ ifs.read (hdr ["ctsize"])
+
+ return objs
+
+
def depdtcrypt (mode, secret, ins, outs):
"""
Remove PDTCRYPT layer from all objects encrypted with the secret. Used on a
return [ slices [i] for i in ovrlp ]
+def mode_ivcheck (ifd):
+ total_obj = 0
+ try:
+ total_obj = check_ivs (ifd)
+ except (NonConsecutiveIV, DuplicateIV) as exn:
+ noise ("PDT: Detected inconsistent initialization vectors")
+ noise ("PDT:")
+ noise ("PDT: “%s”" % exn)
+ noise ("PDT:")
+ noise ("")
+ return 1
+ except Exception as exn:
+ noise ("PDT: Hit an error unrelated to checking IVs")
+ noise ("PDT:")
+ noise ("PDT: “%s”" % exn)
+ noise ("PDT:")
+ return 1
+
+ noise ("PDT: Successfully traversed %d encrypted objects in input."
+ % total_obj)
+ noise ("PDT:")
+ noise ("PDT: All IVs consecutive and unique.")
+
+
def mode_scan (secret, fname, outs=None, nacl=None):
"""
Dissect a binary file, looking for PDTCRYPT headers and objects.
for slice in overlap:
noise ("PDT: × %d→%d" % (slice [0], slice [1]))
+
def usage (err=False):
out = print
if err is True:
out (" %s [ -f | --format ]" % indent)
out ("")
out ("\twhere")
- out ("\t\tSUBCOMMAND main mode: { process | scrypt }")
+ out ("\t\tSUBCOMMAND main mode: { process | scrypt | scan | ivcheck }")
out ("\t\t where:")
out ("\t\t process: extract objects from PDT archive")
out ("\t\t scrypt: calculate hash from password and first object")
+ out ("\t\t scan: scan input for PDTCRYPT headers")
+ out ("\t\t ivcheck: check whether IVs are consecutive")
out ("\t\t-p PASSWORD password to derive the encryption key from")
out ("\t\t-k KEY encryption key as 16 bytes in hexadecimal notation")
out ("\t\t-s enforce strict handling of initialization vectors")
checked_secret (make_secret (key=ek.strip ()))
if secret is None:
- if subcommand == PDTCRYPT_SUB_SCRYPT:
+ if subcommand == PDTCRYPT_SUB_IVCHECK:
+ pass
+ elif subcommand == PDTCRYPT_SUB_SCRYPT:
bail ("ERROR: scrypt hash mode requested but no password given")
elif mode & PDTCRYPT_DECRYPT:
bail ("ERROR: decryption requested but no password given")
bail ("ERROR: input must be seekable; please specify a file")
return True, partial (mode_scan, secret, insspec, outs, nacl=nacl)
+ if subcommand == PDTCRYPT_SUB_IVCHECK:
+ if insspec is None:
+ bail ("ERROR: please supply an input file for checking ivs")
+
if subcommand == PDTCRYPT_SUB_SCRYPT:
if secret [0] == PDTCRYPT_SECRET_KEY:
bail ("ERROR: scrypt mode requires a password")
if insspec is not None or subcommand != PDTCRYPT_SUB_SCRYPT:
ins = deptdcrypt_mk_stream (PDTCRYPT_SOURCE, insspec or "-")
+ if subcommand == PDTCRYPT_SUB_IVCHECK:
+ return True, partial (mode_ivcheck, ins)
+
if subcommand == PDTCRYPT_SUB_SCRYPT:
return True, partial (mode_scrypt, secret [1].encode (), ins, nacl,
fmt=scrypt_format)
git://developer.intra2net.com / python-delta-tar / blobdiff