Implemented deletion of groups of FS objects as single rsync operation
Cleaning of old snapshots uses this feature
Change-Id: Iae2acc1cedf880bdb4f055e04ebc2879d3ef42e5
Partial-Bug: #1513764
diff --git a/rsync_remote.py b/rsync_remote.py
index 0ec4426..e557b35 100644
--- a/rsync_remote.py
+++ b/rsync_remote.py
@@ -110,6 +110,41 @@
self.logger.info('Removing file "{}"'.format(report_name))
return self._rsync_push(source=source, dest=dirname, opts=opts)
+ def rm_all(self, names=[]):
+ '''Remove all files and dirs (recursively) on list as single
+ rsync operation'''
+
+ if type(names) not in (list, tuple):
+ if type(names) is srt:
+ names = [names]
+ else:
+ raise RuntimeError('rsync_remote.rm_all has wrong parameter '
+ '"names" == "{}"'.format(names))
+
+ source = self.url.a_dir(self.tmp.empty_dir)
+
+ # group files by directories
+ dest_dirs = dict()
+ for name in names:
+ dirname, filename = os.path.split(name)
+ if dirname not in dest_dirs.keys():
+ dest_dirs[dirname] = list()
+ dest_dirs[dirname].append(filename)
+
+ for dest_dir, filenames in dest_dirs.items():
+ # prepare filter file for every dest_dir
+ content = ''
+ for filename in filenames:
+ content += '+ {}\n'.format(filename)
+ content += '- *'
+ filter_file = self.tmp.get_file(content=content)
+ # removing specified files on dest_dir
+ self.logger.debug('Removing objects on "{}" directory: {}'
+ ''.format(dest_dir, str(filenames)))
+ opts = "--recursive --delete --filter='merge,p {}'"\
+ "".format(filter_file)
+ self._rsync_push(source=source, dest=dest_dir, opts=opts)
+
def cleandir(self, dirname):
'''Removes directories (recursive) on rsync_url'''
dirname = self.url.a_dir(dirname)
@@ -121,8 +156,7 @@
def rmdir(self, dirname):
'''Removes directories (recursive) on rsync_url'''
self.logger.info('Removing directory "{}"'.format(dirname))
- self.cleandir(dirname)
- return self.rmfile(self.url.a_file(dirname))
+ return self.rm_all(self.url.a_file(dirname))
def mkdir(self, dirname):
'''Creates directories (recirsive, like mkdir -p) on rsync_url'''
diff --git a/trsync.py b/trsync.py
index d9e6923..eed00d3 100644
--- a/trsync.py
+++ b/trsync.py
@@ -82,14 +82,14 @@
try:
# start transaction
result = super(TRsync, self).push(source, repo_path, extra)
- transaction.append(lambda p=repo_path: self.rmdir(p))
+ transaction.append(lambda p=repo_path: self.rm_all(p))
self.logger.info('{}'.format(result))
if save_diff is True:
diff_file = self.tmp.get_file(content='{}'.format(result))
diff_file_name = '{}.diff.txt'.format(repo_path)
super(TRsync, self).push(diff_file, diff_file_name, extra)
- transaction.append(lambda f=diff_file_name: self.rmfile(f))
+ transaction.append(lambda f=diff_file_name: self.rm_all(f))
self.logger.debug('Diff file {} created.'
''.format(diff_file_name))
@@ -99,7 +99,7 @@
self.logger.info('Previous {} -> {}'.format(symlink, tgt))
undo = lambda l=symlink, t=tgt: self.symlink(l, t)
except:
- undo = lambda l=symlink: self.rmfile(l)
+ undo = lambda l=symlink: self.rm_all(l)
# TODO: implement detection of target relative symlink
if symlink.startswith(self.snapshot_dir):
self.symlink(symlink, snapshot_name)
@@ -155,6 +155,8 @@
)
links = self.ls_symlinks(self.url.a_dir())
links += self.ls_symlinks(self.url.a_dir(self.snapshot_dir))
+ snapshots_to_remove = list()
+ new_snapshots = list()
for s in snapshots:
s_date = datetime.datetime.strptime(
s,
@@ -169,11 +171,20 @@
or _[1].endswith('/{}'.format(s))
]
if not s_links:
- self.rmdir(s_path)
- self.rmfile(s_path + '.diff.txt')
+ snapshots_to_remove.append(s_path)
+ snapshots_to_remove.append(s_path + '.diff.txt')
else:
self.logger.info('Skip deletion of "{}" because there are '
'symlinks found: {}'.format(s, s_links))
else:
- self.logger.info('Skip deletion of "{}" because it newer than '
- '{} days'.format(s, save_latest_days))
+ new_snapshots.append(s)
+
+ if new_snapshots:
+ self.logger.info('Skip deletion of snapshots newer than '
+ '{} days: {}'.format(save_latest_days,
+ str(new_snapshots)))
+
+ if snapshots_to_remove:
+ self.logger.info('Removing old snapshots (older then {} days): {}'
+ ''.format(save_latest_days, str(snapshots_to_remove)))
+ self.rm_all(snapshots_to_remove)