Refactored rsync_mirror.TRsync with functional tests
CLI tools updated according with API changes
PEP8 fixes
Closes-Bug: #1570260
Closes-Bug: #1575759
Change-Id: I26b3547854d670da2da94d469f88a3ea057f07db
diff --git a/trsync/cmd/cli.py b/trsync/cmd/cli.py
index eb7fa8f..41997c9 100644
--- a/trsync/cmd/cli.py
+++ b/trsync/cmd/cli.py
@@ -46,11 +46,11 @@
required=False,
help='Specified timestamp will be used for '
'snapshot. Format:yyyy-mm-dd-hhMMSS')
- parser.add_argument('--snapshot-dir',
+ parser.add_argument('--snapshots-dir', '--snapshot-dir',
required=False,
default='snapshots',
- help='Directory name for snapshots. "snapshots" '
- 'by default')
+ help='Directory name for snapshots relative '
+ '"destination". "snapshots" by default')
parser.add_argument('--init-directory-structure',
action='store_true',
required=False,
@@ -58,7 +58,7 @@
help='It specified, all directories including'
'"snapshots-dir" will be created on remote '
'location')
- parser.add_argument('--save-latest-days',
+ parser.add_argument('--snapshot-lifetime', '--save-latest-days',
required=False,
default=61,
help='Snapshots for specified number of days will '
@@ -97,9 +97,9 @@
if properties['extra'].startswith('\\'):
properties['extra'] = properties['extra'][1:]
properties['rsync_extra_params'] = properties.pop('extra')
- properties['save_latest_days'] = \
- None if properties['save_latest_days'] == 'None' \
- else int(properties['save_latest_days'])
+ properties['snapshot_lifetime'] = \
+ None if properties['snapshot_lifetime'] == 'None' \
+ else int(properties['snapshot_lifetime'])
failed = list()
for server in servers:
diff --git a/trsync/cmd/trsync_push.py b/trsync/cmd/trsync_push.py
index 0490ee5..453d207 100755
--- a/trsync/cmd/trsync_push.py
+++ b/trsync/cmd/trsync_push.py
@@ -42,11 +42,11 @@
help='Specified timestamp will be used for snapshot.'
'Format:yyyy-mm-dd-hhMMSS')
- parser.add_argument('--snapshot-dir',
+ parser.add_argument('--snapshots-dir', '--snapshot-dir',
required=False,
default='snapshots',
- help='Directory name for snapshots. "snapshots" '
- 'by default')
+ help='Directory name for snapshots relative '
+ '"destination". "snapshots" by default')
parser.add_argument('--init-directory-structure',
action='store_true',
@@ -55,7 +55,7 @@
help='It specified, all directories including'
'"snapshots-dir" will be created on remote location')
- parser.add_argument('--save-latest-days',
+ parser.add_argument('--snapshot-lifetime', '--save-latest-days',
required=False,
default=61,
help='Snapshots for specified number of days will be '
@@ -99,9 +99,9 @@
if properties['extra'].startswith('\\'):
properties['extra'] = properties['extra'][1:]
properties['rsync_extra_params'] = properties.pop('extra')
- properties['save_latest_days'] = \
- None if options.save_latest_days == 'None' \
- else int(options.save_latest_days)
+ properties['snapshot_lifetime'] = \
+ None if options.snapshot_lifetime == 'None' \
+ else int(options.snapshot_lifetime)
failed = list()
for server in servers:
diff --git a/trsync/objects/rsync_mirror.py b/trsync/objects/rsync_mirror.py
index f7ab0ce..5b6f30e 100644
--- a/trsync/objects/rsync_mirror.py
+++ b/trsync/objects/rsync_mirror.py
@@ -1,53 +1,75 @@
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2015-2016, Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
import datetime
+import logging
import os
from trsync.utils import utils as utils
+from trsync.objects.rsync_ops import RsyncOps
from trsync.objects.rsync_remote import RsyncRemote
from trsync.utils.utils import TimeStamp
+logging.basicConfig()
+log = logging.getLogger(__name__)
+log.setLevel('DEBUG')
+
class TRsync(RsyncRemote):
- # TODO: possible check that rsync url is exists
+ # TODO(mrasskazov): possible check that rsync url is exists
def __init__(self,
rsync_url,
- snapshot_dir='snapshots',
+ snapshots_dir='snapshots',
latest_successful_postfix='latest',
- save_latest_days=14,
+ snapshot_lifetime=14,
init_directory_structure=True,
timestamp=None,
**kwargs
):
super(TRsync, self).__init__(rsync_url, **kwargs)
- self.logger = utils.logger.getChild('TRsync.' + rsync_url)
+ self._log = utils.logger.getChild('TRsync' + rsync_url)
+ self._snapshots_dir = self.url.a_dir(snapshots_dir)
+ self._latest_successful_postfix = latest_successful_postfix
+ self._snapshot_lifetime = snapshot_lifetime
+
self.timestamp = TimeStamp(timestamp)
- self.logger.info('Using timestamp {}'.format(self.timestamp))
- self.snapshot_dir = self.url.a_dir(snapshot_dir)
- self.latest_successful_postfix = latest_successful_postfix
- self.save_latest_days = save_latest_days
+ self._log.info('Using timestamp {}'.format(self.timestamp))
if init_directory_structure is True:
- self.init_directory_structure()
+ super(TRsync, self)._init_directory_structure()
+ self._init_snapshots_dir()
- def init_directory_structure(self):
- dir_full_name = self.url.a_dir(self.url.path, self.snapshot_dir)
- if self.url.url_type != 'path':
- server_root = RsyncRemote(self.url.root)
- return server_root.mkdir(dir_full_name)
- else:
- if not os.path.isdir(dir_full_name):
- return os.makedirs(dir_full_name)
+ def _init_snapshots_dir(self):
+ dir_full_name = self.url.a_dir(self.url.path, self._snapshots_dir)
+ if dir_full_name not in ['', '/']:
+ if self.url.url_type != 'path':
+ rsync_root = RsyncOps(self.url.root)
+ rsync_root.mk_dir(dir_full_name)
+ else:
+ if not os.path.isdir(dir_full_name):
+ os.makedirs(dir_full_name)
return True
-
def push(self, source, repo_name, symlinks=[], extra=None, save_diff=True):
repo_basename = os.path.split(repo_name)[-1]
latest_path = self.url.a_file(
- self.snapshot_dir,
+ self._snapshots_dir,
'{}-{}'.format(self.url.a_file(repo_basename),
- self.latest_successful_postfix)
+ self._latest_successful_postfix)
)
symlinks = list(symlinks)
@@ -56,13 +78,13 @@
snapshot_name = self.url.a_file(
'{}-{}'.format(self.url.a_file(repo_basename), self.timestamp)
)
- repo_path = self.url.a_file(self.snapshot_dir, snapshot_name)
+ repo_path = self.url.a_file(self._snapshots_dir, snapshot_name)
extra = '--link-dest={}'.format(
- self.url.a_file(self.url.path, latest_path)
+ self.url.path_relative(latest_path, repo_path)
)
- # TODO: split transaction run (push or pull), and
+ # TODO(mrasskazov): split transaction run (push or pull), and
# commit/rollback functions. transaction must has possibility to
# rollback after commit for implementation of working with pool
# of servers. should be something like this:
@@ -81,36 +103,42 @@
transaction = list()
try:
# start transaction
- result = super(TRsync, self).push(source, repo_path, extra)
- transaction.append(lambda p=repo_path: self.rm_all(p))
- self.logger.info('{}'.format(result))
+ result = super(TRsync, self).push(self.url.a_dir(source),
+ repo_path,
+ extra)
+ transaction.append(lambda p=repo_path: self.rsync.rm_all(p))
+ self._log.info('{}'.format(result))
if save_diff is True:
- diff_file = self.tmp.get_file(content='{}'.format(result))
+ diff_file = self._tmp.get_file(content='{}'.format(result))
diff_file_name = '{}.diff.txt'.format(repo_path)
- super(TRsync, self).push(diff_file, diff_file_name, extra)
- transaction.append(lambda f=diff_file_name: self.rm_all(f))
- self.logger.debug('Diff file {} created.'
- ''.format(diff_file_name))
+ super(TRsync, self).push(diff_file, diff_file_name)
+ transaction.append(
+ lambda f=diff_file_name: self.rsync.rm_all(f)
+ )
+ self._log.debug('Diff file {} created.'
+ ''.format(diff_file_name))
for symlink in symlinks:
try:
- tgt = [_[1] for _ in self.ls_symlinks(symlink)][0]
- self.logger.info('Previous {} -> {}'.format(symlink, tgt))
- undo = lambda l=symlink, t=tgt: self.symlink(l, t)
- except:
- undo = lambda l=symlink: self.rm_all(l)
- # TODO: implement detection of target relative symlink
- if symlink.startswith(self.snapshot_dir):
- self.symlink(symlink, snapshot_name)
- else:
- self.symlink(symlink, repo_path)
+ tgt = self.rsync.symlink_target(symlink, recursive=False)
+ self._log.info('Previous {} -> {}'.format(symlink, tgt))
+ undo = lambda l=symlink, t=tgt: self.rsync.symlink(l, t)
+ except Exception:
+ undo = lambda l=symlink: self.rsync.rm_all(l)
+ self.rsync.symlink(
+ symlink,
+ self.url.path_relative(
+ os.path.join(self._snapshots_dir, snapshot_name),
+ os.path.split(symlink)[0]
+ )
+ )
transaction.append(undo)
except RuntimeError:
- self.logger.error("Rollback transaction because some of sync"
- "operation failed")
- [_() for _ in reversed(transaction)]
+ self._log.error("Rollback transaction because some of sync"
+ "operation failed")
+ [func() for func in reversed(transaction)]
raise
try:
@@ -118,43 +146,43 @@
# only warning
self._remove_old_snapshots(repo_name)
except RuntimeError:
- self.logger.warn("Old snapshots are not deleted. Ignore. "
- "May be next time.")
+ self._log.warn("Old snapshots are not deleted. Ignore. "
+ "May be next time.")
return result
- def _remove_old_snapshots(self, repo_name, save_latest_days=None):
- if save_latest_days is None:
- save_latest_days = self.save_latest_days
- if save_latest_days is None or save_latest_days is False:
+ def _remove_old_snapshots(self, repo_name, snapshot_lifetime=None):
+ if snapshot_lifetime is None:
+ snapshot_lifetime = self._snapshot_lifetime
+ if snapshot_lifetime is None or snapshot_lifetime is False:
# delete all snapshots
- self.logger.info('Deletion all of the old snapshots '
- '(save_latest_days == {})'
- ''.format(save_latest_days))
- save_latest_days = -1
- elif save_latest_days == 0:
+ self._log.info('Deletion all of the old snapshots '
+ '(snapshot_lifetime == {})'
+ ''.format(snapshot_lifetime))
+ snapshot_lifetime = -1
+ elif snapshot_lifetime == 0:
# skipping deletion
- self.logger.info('Skip deletion of old snapshots '
- '(save_latest_days == {})'
- ''.format(save_latest_days))
+ self._log.info('Skip deletion of old snapshots '
+ '(snapshot_lifetime == {})'
+ ''.format(snapshot_lifetime))
return
else:
# delete snapshots older than
- self.logger.info('Deletion all of the unlinked snapshots older '
- 'than {0} days (save_latest_days == {0})'
- ''.format(save_latest_days))
+ self._log.info('Deletion all of the unlinked snapshots older '
+ 'than {0} days (snapshot_lifetime == {0})'
+ ''.format(snapshot_lifetime))
warn_date = \
- self.timestamp.now - datetime.timedelta(days=save_latest_days)
+ self.timestamp.now - datetime.timedelta(days=snapshot_lifetime)
warn_date = datetime.datetime.combine(warn_date, datetime.time(0))
- snapshots = self.ls_dirs(
- self.url.a_dir(self.snapshot_dir),
+ snapshots = self.rsync.ls_dirs(
+ self.url.a_dir(self._snapshots_dir),
pattern=r'^{}-{}$'.format(
repo_name,
self.timestamp.snapshot_stamp_pattern
)
)
- links = self.ls_symlinks(self.url.a_dir())
- links += self.ls_symlinks(self.url.a_dir(self.snapshot_dir))
+ links = self.rsync.ls_symlinks(self.url.a_dir())
+ links += self.rsync.ls_symlinks(self.url.a_dir(self._snapshots_dir))
snapshots_to_remove = list()
new_snapshots = list()
for s in snapshots:
@@ -164,7 +192,7 @@
self.timestamp.snapshot_stamp_format)
)
s_date = datetime.datetime.combine(s_date, datetime.time(0))
- s_path = self.url.a_file(self.snapshot_dir, s)
+ s_path = self.url.a_file(self._snapshots_dir, s)
if s_date < warn_date:
s_links = [_[0] for _ in links
if _[1] == s
@@ -174,17 +202,18 @@
snapshots_to_remove.append(s_path)
snapshots_to_remove.append(s_path + '.diff.txt')
else:
- self.logger.info('Skip deletion of "{}" because there are '
- 'symlinks found: {}'.format(s, s_links))
+ self._log.info('Skip deletion of "{}" because there are '
+ 'symlinks found: {}'.format(s, s_links))
else:
new_snapshots.append(s)
if new_snapshots:
- self.logger.info('Skip deletion of snapshots newer than '
- '{} days: {}'.format(save_latest_days,
- str(new_snapshots)))
+ self._log.info('Skip deletion of snapshots newer than '
+ '{} days: {}'.format(snapshot_lifetime,
+ str(new_snapshots)))
if snapshots_to_remove:
- self.logger.info('Removing old snapshots (older then {} days): {}'
- ''.format(save_latest_days, str(snapshots_to_remove)))
- self.rm_all(snapshots_to_remove)
+ self._log.info('Removing old snapshots (older then {} days): {}'
+ ''.format(snapshot_lifetime,
+ str(snapshots_to_remove)))
+ self.rsync.rm_all(snapshots_to_remove)
diff --git a/trsync/tests/test_rsync_mirror.py b/trsync/tests/test_rsync_mirror.py
new file mode 100644
index 0000000..f698662
--- /dev/null
+++ b/trsync/tests/test_rsync_mirror.py
@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2015-2016, Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+
+from time import sleep
+
+from trsync.objects.rsync_mirror import TRsync
+from trsync.tests import rsync_base
+from trsync.utils.tempfiles import TempFiles
+
+
+logging.basicConfig()
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+
+class TestRsyncMirror(rsync_base.TestRsyncBase):
+
+ """Test case class for rsync_mirror module"""
+
+ def test__init_directory_structure(self):
+ for remote in self.rsyncd[self.testname]:
+ path = '/initial_test_path/test-subdir'
+ url = remote.url + path
+ rsync = TRsync(url, init_directory_structure=True)
+ self.assertTrue(os.path.isdir(remote.path + path + '/snapshots'))
+ del rsync
+
+ def test_push(self):
+ for remote in self.rsyncd[self.testname]:
+ # create test data file
+ temp_dir = TempFiles()
+ src_dir = temp_dir.last_temp_dir
+ self.getDataFile(os.path.join(src_dir,
+ 'dir1/dir2/dir3/test_data.txt'))
+
+ # First snapshot
+ rsync = TRsync(remote.url)
+ out = rsync.push(os.path.join(src_dir, 'dir1'), 'dir1')
+ timestamp1 = rsync.timestamp.snapshot_stamp
+ snapshot1_path = remote.path + '/snapshots/dir1-{}'\
+ ''.format(timestamp1)
+ latest_path = remote.path + '/snapshots/dir1-latest'
+ self.assertDirsEqual(snapshot1_path, src_dir + '/dir1')
+ self.assertTrue(os.path.isdir(remote.path + '/snapshots/'))
+ self.assertTrue(os.path.islink(latest_path))
+ self.assertEqual(snapshot1_path, os.path.realpath(latest_path))
+ with open(snapshot1_path + '.diff.txt') as diff_file:
+ self.assertEqual(out, diff_file.read())
+ with open(latest_path + '.target.txt') as target_file:
+ self.assertEqual(
+ ['dir1-' + timestamp1],
+ target_file.read().splitlines()
+ )
+ # test_data.txt has only one hardlinks
+ self.assertEqual(
+ 1,
+ os.stat(os.path.join(snapshot1_path,
+ 'dir2/dir3/test_data.txt')).st_nlink
+ )
+
+ # Second snapshot
+ sleep(1)
+ rsync = TRsync(remote.url)
+ out = rsync.push(os.path.join(src_dir, 'dir1'), 'dir1')
+ timestamp2 = rsync.timestamp.snapshot_stamp
+ self.assertNotEqual(timestamp1, timestamp2)
+ snapshot2_path = remote.path + '/snapshots/dir1-{}'\
+ ''.format(timestamp2)
+ self.assertDirsEqual(snapshot2_path, src_dir + '/dir1')
+ self.assertDirsEqual(snapshot1_path, snapshot2_path)
+ self.assertTrue(os.path.islink(latest_path))
+ self.assertEqual(snapshot2_path, os.path.realpath(latest_path))
+ with open(snapshot2_path + '.diff.txt') as diff_file:
+ self.assertEqual(out, diff_file.read())
+ with open(latest_path + '.target.txt') as target_file:
+ self.assertEqual(
+ ['dir1-' + timestamp2,
+ 'dir1-' + timestamp1],
+ target_file.read().splitlines()
+ )
+ # test_data.txt has two hardlinks in both snapshots
+ self.assertEqual(
+ 2,
+ os.stat(os.path.join(snapshot1_path,
+ 'dir2/dir3/test_data.txt')).st_nlink
+ )
+ self.assertEqual(
+ 2,
+ os.stat(os.path.join(snapshot2_path,
+ 'dir2/dir3/test_data.txt')).st_nlink
+ )
+
+ # TODO(mrasskazov) implement VVV
+ # Push to existent snapshot raise RuntimeError
+ # rsync = TRsync(remote.url, timestamp=timestamp1)
+ # self.assertRaises(
+ # RuntimeError,
+ # rsync.push,
+ # os.path.join(src_dir, 'dir1'),
+ # 'dir1'
+ # )
+ # CLI parameters: --raise-if-snapshot-exists,
+ # --update-existent-snapshot
+
+ # TODO(mrasskazov) implement VVV
+ # Push new snapshot during locked remote mirror - wait for finish
+ # previous operation or fail (optional)
+ # CLI parameters: --raise-if-locked, --wait-if-locked,
+ # --ignore-locking