From da29ba19f4231dbdecf021ff7374fff9a30d15c8 Mon Sep 17 00:00:00 2001 From: Eduardo Robles Elvira Date: Wed, 24 Jul 2013 09:53:00 +0200 Subject: [PATCH] improving the design of DeltaTar public API, making it more flexible --- docs/design.py | 147 +++++++++++++++++++++++++++++++++++--------------------- 1 files changed, 92 insertions(+), 55 deletions(-) diff --git a/docs/design.py b/docs/design.py index c086f62..be074aa 100644 --- a/docs/design.py +++ b/docs/design.py @@ -4,36 +4,28 @@ Backup Files Index format: * it will contain one line per file in the directory, even if the file didn't change. This way we can restore a diff backup without needing previous diffs. -{"path": value, "stat.st_mode": value, "mtime": value, "ctime": value, "uid": value, "gid": value, "inode": value, "size": value} -{"path": value, "stat.st_mode": value, "mtime": value, "ctime": value, "uid": value, "gid": value, "inode": value, "size": value} +{"path": value, "stat.st_mode": value, "mtime": value, "ctime": value, "uid": value, "gid": value, "inode": value, "size": value, "volume": 0, "offset": 0} +{"path": value, "stat.st_mode": value, "mtime": value, "ctime": value, "uid": value, "gid": value, "inode": value, "size": value, "volume": 0, "offset": 56464} [...] -{"path": value, "stat.st_mode": value, "mtime": value, "ctime": value, "uid": value, "gid": value, "inode": value, "size": value} +{"path": value, "stat.st_mode": value, "mtime": value, "ctime": value, "uid": value, "gid": value, "inode": value, "size": value, "volume": 1, "offset": 0} -''' - -''' DeltaTar proposed backup directory structure is quite simple: backups/ -├── 1/ -│   ├── backup_index -│   ├── backup.tar.gz.aes128 -│   ├── backup.tar.gz.aes128.1 -│   ├── backup.tar.gz.aes128.2 -│   └── backup.tar.gz.aes128.3 -├── 2_label/ -│   ├── backup_index -│   ├── backup.tar.gz.aes128 -└── 3/ -│   ├── backup_index - ├── backup.tar.gz.aes128 - ├── backup.tar.gz.aes128.1 - -Each subdir corresponds with a specific backup. The ctime of the dir indicates -when the backup was done and it also contains the optional label (2_label). - -The format is kept quite simple so that each subdir can easily be used to do a -restore independently. +├── backup-2013-07-22-0200/ +│   ├── bfull-2013-07-22-0200.index +│   ├── bfull-2013-07-22-0200.tar.gz.aes128 +│   ├── bfull-2013-07-22-0200-001.tar.gz.aes128 +│   ├── bfull-2013-07-22-0200-002.tar.gz.aes128 +│   └── bfull-2013-07-22-0200-003.tar.gz.aes128 +├── backup-2013-07-22-1400/ +│   ├── bdiff-2013-07-22-0200.index +│   ├── bdiff-2013-07-22-0200.tar.gz.aes128 +└── backup-2013-07-23-0200/ +│   ├── bdiff-2013-07-22-0200.index +│   ├── bdiff-2013-07-22-0200.tar.gz.aes128 +│   ├── bdiff-2013-07-22-0200-001.tar.gz.aes128 + ''' @@ -44,7 +36,8 @@ class DeltaTar(object): def __init__(self, excluded_files=[], max_volume_size, included_files=[], filter_func=None, mode="tar", - index_encrypted=True, password=None, log_path=None): + password=None, log_path=None, index_encrypted=True, + index_name_func=None, volume_name_func=None): ''' Constructor. Configures the diff engine. @@ -63,70 +56,114 @@ class DeltaTar(object): - mode: Mode in which the delta will be created. Accepts the same modes as our tarfile python library. - - index_encrypted: whether the index should be encrypted or not. Only - makes sense to set it as True if mode includes aes128 or aes256 - - password: used together with aes modes to encrypt and decrypt backups. - log_path: creates the backup log in this path. + + - index_encrypted: whether the index should be encrypted or not. Only + makes sense to set it as True if mode includes aes128 or aes256. + + - index_name_func: function that sets a custom name for the index file. This + function receives the backup_path and if it's a full backup as arguments + and must return the name of the corresponding index file. Optional, + DeltaTar gives index files a "backup.index" name by default. + + - volume_name_func: function that defines the name of tar volumes. It + receives the backup_path, if it's a full backup and the volume number, + and must return the name for the corresponding volume name. Optional, + DeltaTar has default names for tar volumes. ''' pass - def create_backup(self, source_path, backup_path, label=None): + def create_full_backup(self, source_path, backup_path): + ''' + Creates a full backup. + + Parameters: + - source_path: source path to the directory to back up. + - backup_path: path where the back up will be stored. Backup path will + be created if not existent. + ''' + pass + + def create_diff_backup(self, source_path, backup_path, previous_index_path): ''' Creates a backup. Parameters: - source_path: source path to the directory to back up. - - backup_path: path where the back up will be stored. - - label: optional label for the backup. + - backup_path: path where the back up will be stored. Backup path will + be created if not existent. + - previous_index_path: index of the previous backup, needed to know + which files changed since then. ''' pass - def restore_backup(self, backup_path, target_path, restore_label=None, - restore_date=None): + def restore_backup(self, target_path, backup_indexes_paths=[]): ''' Restores a backup. Parameters: - backup_path: path where the back up will is stored. - target_path: path to restore. - - restore_label: label to be restored, optional. - - restore_date: date to be restored, optional. + - backup_indexes_paths: path to backup indexes, in descending date order. + The indexes indicate the location of their respective backup volumes, + and multiple indexes are needed to be able to restore diff backups. ''' + pass + class TestDeltaTar(UnitTest): ''' This is an example of how DeltaTar class could be used ''' def test_create(self): + import os + from deltatar import DeltaTar + + def index_name_func(backup_path, is_full): + prefix = "bfull" if is_full else "bdiff" + # get the name and remove backup- + basename = os.path.basename(backup_path)[7:] + + return "%s-%s.index" % (prefix, basename) + + def volume_name_func(backup_path, is_full, volume_number): + ''' + Handles the new volumes + ''' + prefix = "bfull" if is_full else "bdiff" + # get the name and remove backup- + basename = os.path.basename(backup_path)[7:] + + return "%s-%s-%03d.tar.gz.aes128" % (prefix, basename, volume_number) + + # constructor of DeltaTar class allows to set the configuration deltatar = DeltaTar( # these options are the same as in tarfile: - mode="tar#gz.aes128" + mode="tar#gz.aes128", max_volume_size=100*1024*1024, # 100MB - ) + index_name_func=index_name_func, # optional + volume_name_func=volume_name_func # optional + ) # create first backup - deltatar.create_backup( - source_path="/path/to/important/dir/", - backup_path="/var/backups/") + deltatar.create_full_backup( + source_path="/path/to/important/dir", + backup_path="/var/backups/backup-2013-07-22-0200") - # here: change some files + # here: change some files # create second backup - deltatar.create_backup( - source_path="/path/to/important/dir/", - backup_path="/var/backups/", - label="labelled backup") + deltatar.create_diff_backup( + source_path="/path/to/important/dir", + backup_path="/var/backups/backup-2013-07-22-1400", + previous_index_path="/var/backups/backup-2013-07-22-0200/bfull-2013-07-22-0200.index") # restore backup in another dir. it will restore last version - deltatar.restore_backup( - backup_path="/var/backups/", - target_path="/path/to/second/dir/") - - # roll back to the second backup - deltatar.restore_backup( - backup_path="/var/backups/", - target_path="/path/to/second/dir/", - restore_label="labelled backup") + deltatar.restore_backup(target_path="/path/to/second/dir", + backup_indexes_paths=[ + "/var/backups/backup-2013-07-22-1400/bfull-2013-07-22-1400.index", + "/var/backups/backup-2013-07-22-0200/bfull-2013-07-22-0200.index" + ]) -- 1.7.1