allow for numbers of missing and failed files to differ in recovery test

[python-delta-tar] / backup.py

#!/usr/bin/env python3

# Copyright (C) 2013 Intra2net AG
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.html>

import argparse
import binascii
import json
import logging
import os
import re
import random
import shutil
from datetime import datetime
from functools import partial

from deltatar.deltatar import DeltaTar, NO_MATCH, MATCH, PARENT_MATCH
from deltatar.tarfile import TarFile, GNU_FORMAT

consoleLogger = logging.StreamHandler()
consoleLogger.setLevel(logging.DEBUG)

def check_equal_dirs(path1, path2, deltatar):
    '''
    compare the two directories source_dir and target_dir and check
    # they are the same
    '''
    nbars1 = len(path1.split('/'))
    nbars2 = len(path2.split('/'))

    source_it = deltatar._recursive_walk_dir(path1)
    source_it = deltatar.jsonize_path_iterator(source_it, strip=nbars1)
    target_it = deltatar._recursive_walk_dir(path2)
    target_it = deltatar.jsonize_path_iterator(target_it, strip=nbars2)
    while True:
        try:
            sitem = next(source_it)
            titem = next(target_it)
        except StopIteration:
            try:
                titem = next(target_it)
                raise Exception("iterators do not stop at the same time")
            except StopIteration:
                break
        try:
            assert deltatar._equal_stat_dicts(sitem, titem)
        except Exception as e:
            print(sitem)
            print(titem)
            print("FAIL")
            raise

if __name__ == "__main__":
    #import tracemalloc
    #tracemalloc.enable()
    #top = tracemalloc.DisplayTop(25)
    #top.show_lineno = True
    #top.start(20)
    parser = argparse.ArgumentParser()

    parser.add_argument("-m", "--mode", default='',
                        help=
                            """Mode in which to read/write the backup.
                            Valid modes are:
                                       ''           open uncompressed;
                                       'gz'         open with gzip compression;
                                       'bz2'        open with bzip2 compression;
                                       '#gz'        encrypted stream of individually
                                                    compressed tar blocks.
                            To enable encryption, supply a password.
                            """)
    parser.add_argument("-t", "--targetpath", help="Target path directory.")
    parser.add_argument("-s", "--sourcepath", help="Source path directory.")
    epw = os.getenv ("PDTCRYPT_PASSWORD")
    parser.add_argument("-p", "--password",
                        default=epw.strip () if epw is not None else None,
                        help="Password for symmetric encryption. "
                             "The environment variable PDTCRYPT_PASSWORD should "
                             "be preferred to this. Enables encryption.")
    parser.add_argument("-v", "--volsize", default=None,
                        help="Maximum volume size, in megabytes.")
    parser.add_argument("-r", "--restore", action='store_true',
                        help="Restore a backup.")
    parser.add_argument("-R", "--recover", action='store_true',
                        help="Restore a backup with an index file.")
    parser.add_argument("-f", "--full", action='store_true',
                        help="Create a full backup.")
    parser.add_argument("-d", "--diff", action='store_true',
                        help="Create a diff backup.")
    parser.add_argument("-i", "--indexes", nargs='+', help="Indexes paths.")
    parser.add_argument("-l", "--list-files", action='store_true',
                        help="List files in a tarball.")
    parser.add_argument("-x", "--excluded", nargs='+', default=[],
                        help="Files to exclude from backup.")
    parser.add_argument("-inc", "--included", nargs='+', default=[],
                        help="Files to include in the backup.")
    parser.add_argument("-ip", "--included-path", default=None,
                        help="Path to a file containing a list of included paths.")
    parser.add_argument("-xp", "--excluded-path", default=None,
                        help="Path to a file containing a list of excluded paths.")
    parser.add_argument("-e", "--equals", action='store_true',
                        help="Check if two dirs are equal.")

    args = parser.parse_args()

    if args.excluded_path:
        f = open(args.excluded_path, 'r')
        excluded_files = f.readlines()
        f.close()
    else:
        excluded_files = args.excluded

    if args.included_path:
        f = open(args.included_path, 'r')
        included_files = f.readlines()
        f.close()
    else:
        included_files = args.included

    deltatar = DeltaTar(mode=args.mode, password=args.password,
        logger=consoleLogger, excluded_files=excluded_files,
        included_files=included_files)

    if args.full:
        deltatar.create_full_backup(args.sourcepath, args.targetpath, args.volsize)
    elif args.diff:
        deltatar.create_diff_backup(args.sourcepath, args.targetpath, args.indexes[0], args.volsize)
    elif args.list_files:
        deltatar.list_backup(args.sourcepath, lambda e: print (e.name))
    elif args.restore:
        if args.indexes is not None:
            deltatar.restore_backup(args.targetpath, backup_indexes_paths=args.indexes)
        else:
            deltatar.restore_backup(args.targetpath, backup_tar_path=args.sourcepath)
    elif args.recover:
        failed = deltatar.recover_backup(args.targetpath,
                                         backup_indexes_paths=args.indexes,
                                         backup_tar_path=args.sourcepath)
        if len (failed) > 0:
            logger = logging.getLogger('deltatar.DeltaTar')
            print ("%d files could not be restored:" % len (failed))
            for i, f in enumerate (failed):
                print ("   [%d] %s (%s)" % (i, f [0], f [1]))

    elif args.equals:
        check_equal_dirs(os.path.abspath(args.sourcepath), os.path.abspath(args.targetpath), deltatar)
    else:
        import sys
        print("Nothing to do.\nPlease specify one of --full, --diff, --list-files, "
              "--restore, --equals.\n", file=sys.stderr)
        parser.print_help(file=sys.stderr)