From: Philipp Gesang Date: Thu, 4 May 2017 16:06:04 +0000 (+0200) Subject: move final IV checks out of crypto context X-Git-Tag: v2.2~7^2~131 X-Git-Url: http://developer.intra2net.com/git/?a=commitdiff_plain;h=3031b7ae605eb641046fb3fda13227e88870a2e1;p=python-delta-tar move final IV checks out of crypto context Collect IVs while decrypting but postpone the final check for duplicates. Reused IVs still trigger an exception during decryption but since multiple different contexts may be active (e. g. when handling a diff backup), the IVs they retrieved from the headers must be compared afterwards. This test has its place in a new function “validate” of the ``RestoreHelper`` and must be called when decryption has been completed. --- diff --git a/deltatar/crypto.py b/deltatar/crypto.py index cec61e0..44ca91f 100755 --- a/deltatar/crypto.py +++ b/deltatar/crypto.py @@ -83,7 +83,8 @@ import cryptography __all__ = [ "hdr_make", "hdr_read", "hdr_fmt", "hdr_fmt_pretty" - , "PDTCRYPT_HDR_SIZE" , "AES_GCM_IV_CNT_DATA", "AES_GCM_IV_CNT_INFOFILE" + , "PDTCRYPT_HDR_SIZE", "AES_GCM_IV_CNT_DATA" + , "AES_GCM_IV_CNT_INFOFILE", "AES_GCM_IV_CNT_INDEX" ] @@ -216,8 +217,11 @@ AES_GCM_IV_LEN = 12 AES_GCM_MAX_SIZE = (1 << 36) - (1 << 5) # 2^39 - 2^8 b ≅ 64 GB AES_GCM_FMT_TAG = "<16s" +# index and info files are written on-the fly while encrypting so their +# counters must be available inadvance AES_GCM_IV_CNT_INFOFILE = 1 # constant -AES_GCM_IV_CNT_DATA = AES_GCM_IV_CNT_INFOFILE + 1 # also for multivolume +AES_GCM_IV_CNT_INDEX = AES_GCM_IV_CNT_INFOFILE + 1 +AES_GCM_IV_CNT_DATA = AES_GCM_IV_CNT_INDEX + 1 # also for multivolume AES_GCM_IV_CNT_MAX = 0xffFFffFF @@ -323,6 +327,18 @@ def hex_spaced_of_bytes (b): + (len (b) | 1 == len (b) and " %.2x" % b[-1] or "") # odd lengths +def hdr_iv_counter (h): + """Extract the variable part of the IV of the given header.""" + _fixed, cnt = struct.unpack (FMT_I2N_IV, h ["iv"]) + return cnt + + +def hdr_iv_fixed (h): + """Extract the fixed part of the IV of the given header.""" + fixed, _cnt = struct.unpack (FMT_I2N_IV, h ["iv"]) + return fixed + + hdr_dump = hex_spaced_of_bytes @@ -410,10 +426,6 @@ def kdf_by_version (paramversion): return partial (fn, params) -STATE_FRESH = 0 -STATE_DEAD = 1 -STATE_LIVE = 2 - class Crypto (object): """ Encryption context to remain alive throughout an entire tarfile pass. @@ -424,6 +436,7 @@ class Crypto (object): cnt = None # file counter (uint32_t != 0) iv = None # current IV fixed = None # accu for 64 bit fixed parts of IV + used_ivs = None # tracks IVs during decryption password = None paramversion = None stats = { "in" : 0 @@ -432,7 +445,8 @@ class Crypto (object): ctsize = -1 ptsize = -1 - info_counter_used = False + info_counter_used = False + index_counter_used = False def __init__ (self, *al, **akv): self.set_parameters (*al, **akv) @@ -455,7 +469,11 @@ class Crypto (object): raise InvalidParameter ("attempted to reuse info file counter " "%d: must be unique" % cnt) self.info_counter_used = True - return + elif cnt == AES_GCM_IV_CNT_INDEX: + if self.index_counter_used is True: + raise InvalidParameter ("attempted to reuse index file counter " + "%d: must be unique" % cnt) + self.index_counter_used = True if cnt <= AES_GCM_IV_CNT_MAX: self.cnt = cnt return @@ -558,11 +576,12 @@ class Encrypt (Crypto): if isinstance (filename, str) is False: raise InvalidParameter ("next: filename must be a string, no %s" % type (filename)) - if counter is not None \ - and isinstance (counter, int) is False: - raise InvalidParameter ("next: the supplied counter is of invalid " - "type %s; please pass an integer instead" - % type (counter)) + if counter is not None: + if isinstance (counter, int) is False: + raise InvalidParameter ("next: the supplied counter is of " + "invalid type %s; please pass an " + "integer instead" % type (counter)) + self.set_object_counter (counter) self.iv = self.iv_make () if self.paramenc == "aes-gcm": self.enc = Cipher \ @@ -579,7 +598,7 @@ class Encrypt (Crypto): self.lastinfo = (filename, hdrdum) super().next (self.password, self.paramversion, self.nacl) - self.set_object_counter (counter if counter is not None else self.cnt + 1) + self.set_object_counter (self.cnt + 1) return hdrdum @@ -615,9 +634,9 @@ class Encrypt (Crypto): class Decrypt (Crypto): - tag = None # GCM tag, part of header - used_ivs = None # if a set, panic on duplicate object IV - last_iv = None # check consecutive ivs in strict mode + tag = None # GCM tag, part of header + strict_ivs = False # if True, panic on duplicate object IV + last_iv = None # check consecutive ivs in strict mode def __init__ (self, password, counter=None, fixedparts=None, strict_ivs=False): @@ -638,8 +657,8 @@ class Decrypt (Crypto): self.fixed.sort () super().__init__ (password, counter=counter) - if strict_ivs is True: - self.used_ivs = set () + self.used_ivs = set () + self.strict_ivs = strict_ivs super().__init__ (password, counter=counter) @@ -652,7 +671,7 @@ class Decrypt (Crypto): def check_duplicate_iv (self, iv): - if iv in self.used_ivs: + if self.strict_ivs is True and iv in self.used_ivs: raise DuplicateIV ("iv [%r] was reused" % iv) # vi has not been used before; add to collection self.used_ivs.add (iv) @@ -660,7 +679,8 @@ class Decrypt (Crypto): def check_consecutive_iv (self, iv): fixed, cnt = struct.unpack (FMT_I2N_IV, iv) - if self.last_iv is not None \ + if self.strict_ivs is True \ + and self.last_iv is not None \ and self.last_iv [0] == fixed \ and self.last_iv [1] != cnt - 1: raise NonConsecutiveIV ("iv [%r] counter not successor of " @@ -691,9 +711,8 @@ class Decrypt (Crypto): fixed, _ = struct.unpack (FMT_I2N_IV, iv) raise InvalidIVFixedPart ("iv [%r] has invalid fixed part [%r]" % (iv, fixed)) - if self.used_ivs is not None: - self.check_duplicate_iv (iv) - self.check_consecutive_iv (iv) + self.check_duplicate_iv (iv) + self.check_consecutive_iv (iv) self.tag = tag defs = ENCRYPTION_PARAMETERS.get (paramversion, None) diff --git a/deltatar/deltatar.py b/deltatar/deltatar.py index 62ad560..b5a73a2 100644 --- a/deltatar/deltatar.py +++ b/deltatar/deltatar.py @@ -557,16 +557,17 @@ class DeltaTar(object): enccounter = None if mode == "w": crypto_ctx = self.initialize_encryption (CRYPTO_MODE_ENCRYPT) - if crypto_ctx is not None: - if kind == AUXILIARY_FILE_INFO: - enccounter = crypto.AES_GCM_IV_CNT_INFOFILE - elif kind == AUXILIARY_FILE_INDEX: - enccounter = crypto.AES_GCM_IV_CNT_INDEX - else: - raise Exception ("invalid kind of aux file %r" % kind) elif mode == "r": crypto_ctx = self.initialize_encryption (CRYPTO_MODE_DECRYPT) + if crypto_ctx is not None: + if kind == AUXILIARY_FILE_INFO: + enccounter = crypto.AES_GCM_IV_CNT_INFOFILE + elif kind == AUXILIARY_FILE_INDEX: + enccounter = crypto.AES_GCM_IV_CNT_INDEX + else: + raise Exception ("invalid kind of aux file %r" % kind) + sink = tarfile._Stream(name=path, mode=mode, comptype=comptype, bufsize=tarfile.RECORDSIZE, fileobj=None, encryption=crypto_ctx, enccounter=enccounter) @@ -809,12 +810,13 @@ class DeltaTar(object): volume_number=0) tarfile_path = os.path.join(backup_path, vol_name) - # postpone creation of index file to accomodate encryption - index_accu = io.BytesIO () - # init index cwd = os.getcwd() + index_name = self.index_name_func(is_full=False) + index_path = os.path.join(backup_path, index_name) + index_sink = self.open_auxiliary_file(index_path, 'w') + def new_volume_handler(deltarobj, cwd, backup_path, tarobj, base_name, volume_number): ''' Handles the new volumes @@ -834,12 +836,12 @@ class DeltaTar(object): # wraps some args from context into the handler new_volume_handler = partial(new_volume_handler, self, cwd, backup_path) - index_accu.write(bytes('{"type": "python-delta-tar-index", "version": 1, "backup-type": "diff", "extra_data": %s}\n' % extra_data_str, 'UTF-8')) + index_sink.write(bytes('{"type": "python-delta-tar-index", "version": 1, "backup-type": "diff", "extra_data": %s}\n' % extra_data_str, 'UTF-8')) s = bytes('{"type": "BEGIN-FILE-LIST"}\n', 'UTF-8') # calculate checksum and write into the stream crc = binascii.crc32(s) & 0xFFFFffff - index_accu.write(s) + index_sink.write(s) # start creating the tarfile tarobj = tarfile.TarFile.open(tarfile_path, @@ -934,22 +936,17 @@ class DeltaTar(object): # store the stat dict in the index s = bytes(json.dumps(stat) + '\n', 'UTF-8') crc = binascii.crc32(s, crc) & 0xffffffff - index_accu.write(s) + index_sink.write(s) s = bytes('{"type": "END-FILE-LIST"}\n', 'UTF-8') crc = binascii.crc32(s, crc) & 0xffffffff - index_accu.write(s) + index_sink.write(s) s = bytes('{"type": "file-list-checksum", "checksum": %d}\n' % crc, 'UTF-8') - index_accu.write(s) + index_sink.write(s) index_it.release() os.chdir(cwd) tarobj.close() - - index_name = self.index_name_func(is_full=False) - index_path = os.path.join(backup_path, index_name) - index_sink = self.open_auxiliary_file(index_path, 'w') - index_sink.write(index_accu.getvalue ()) index_sink.close() @@ -1416,6 +1413,7 @@ class DeltaTar(object): index_it.release() os.chdir(cwd) helper.cleanup() + helper.validate() def _parse_json_line(self, f, l_no): ''' @@ -1469,7 +1467,8 @@ class RestoreHelper(object): self._directories = [] self._deltatar = deltatar self._cwd = cwd - self.password = deltatar.password + self._password = deltatar.password + self._decryptors = [] try: import grp, pwd @@ -1486,8 +1485,8 @@ class RestoreHelper(object): is_full = index == index_list[-1] decryptor = None - if self.password is not None: - decryptor = crypto.Decrypt (self.password) + if self._password is not None: + decryptor = crypto.Decrypt (self._password) # make paths absolute to avoid cwd problems if not os.path.isabs(index): @@ -1533,6 +1532,24 @@ class RestoreHelper(object): ) self._data.append(s) + + def validate (self): + """If encryption was used, verify post-conditions.""" + if len (self._decryptors) == 0: + return + acc = None + for dec in self._decryptors: + if acc is None: + acc = dec.used_ivs.copy () + else: + used_ivs = dec.used_ivs + intersect = used_ivs & acc + if len (intersect) > 0: + raise Exception ("ERROR: %d duplicate IVs found during " + "decryption" % len (intersect)) + acc |= used_ivs + + def cleanup(self): ''' Closes all open files diff --git a/deltatar/tarfile.py b/deltatar/tarfile.py index 3d91e43..50b395f 100644 --- a/deltatar/tarfile.py +++ b/deltatar/tarfile.py @@ -452,7 +452,16 @@ class _Stream: A stream-like object could be for example: sys.stdin, sys.stdout, a socket, a tape device etc. - _Stream is intended to be used only internally. + _Stream is intended to be used only internally but is + nevertherless used externally by Deltatar. + + When encrypting, the ``enccounter`` will be used for + initializing the first cryptographic context. When + decrypting, its value will be compared to the decrypted + object. Decryption fails if the value does not match. + In effect, this means that a ``_Stream`` whose ctor was + passed ``enccounter`` can only be used to encrypt or + decrypt a single object. """ remainder = -1 # track size in encrypted entries @@ -477,6 +486,10 @@ class _Stream: if comptype == '': comptype = "tar" + self.enccounter = None + if self.arcmode & ARCMODE_ENCRYPT: + self.enccounter = enccounter + self.name = name or "" self.mode = mode self.comptype = comptype @@ -510,7 +523,7 @@ class _Stream: elif mode == "w": if not (self.arcmode & ARCMODE_CONCAT): if self.arcmode & ARCMODE_ENCRYPT: - self._init_write_encrypt (name, enccounter) + self._init_write_encrypt (name) self._init_write_gz () self.crc = zlib.crc32(b"") & 0xFFFFffff @@ -548,7 +561,7 @@ class _Stream: if not (self.arcmode & ARCMODE_CONCAT) \ and mode == "w" \ and self.arcmode & ARCMODE_ENCRYPT: - self._init_write_encrypt (name, enccounter) + self._init_write_encrypt (name) else: if self.arcmode & ARCMODE_ENCRYPT: @@ -827,6 +840,13 @@ class _Stream: "processing %r at pos %d" % (exn, self.fileobj, lasthdr)) \ from exn + if self.enccounter is not None: + # enforce that the iv counter in the header matches an + # explicitly requested one + iv = crypto.hdr_iv_counter (hdr) + if iv != self.enccounter: + raise DecryptionError ("expected IV counter %d, got %d" + % (self.enccounter, iv)) self.lasthdr = lasthdr self.remainder = hdr ["ctsize"] # distance to next header self.encryption.next (hdr)