| Max Rasskazov | 8608943 | 2015-06-04 22:07:20 +0300 | [diff] [blame] | 1 | #-*- coding: utf-8 -*- |
| 2 | |
| 3 | import os |
| 4 | import re |
| 5 | |
| 6 | import utils |
| 7 | |
| 8 | from tempfiles import TempFiles |
| 9 | from shell import Shell |
| 10 | from rsync_url import RsyncUrl |
| 11 | |
| 12 | |
| 13 | class RsyncRemote(object): |
| 14 | def __init__(self, |
| 15 | rsync_url, |
| 16 | rsync_extra_params='-v --no-owner --no-group', |
| 17 | ): |
| 18 | # TODO: retry parameters for rsync |
| 19 | self.logger = utils.logger.getChild('RsyncRemote.' + rsync_url) |
| 20 | self.tmp = TempFiles() |
| 21 | self.shell = Shell(self.logger) |
| 22 | self.root = RsyncUrl(rsync_url) |
| 23 | self.rsync_extra_params = rsync_extra_params |
| 24 | |
| 25 | def _do_rsync(self, source='', dest=None, opts='', extra=None): |
| 26 | # TODO: retry for rsync |
| 27 | # TODO: locking: |
| 28 | # https://review.openstack.org/#/c/147120/4/utils/simple_http_daemon.py |
| 29 | # create lock-files on remotes during operations |
| 30 | # for reading and writing |
| 31 | # special option for ignore lock-files (for manual fixing) |
| 32 | # all high-level functions (like ls) specify type of lock(read or |
| 33 | # write), and _do_rsync creates special lock file on remote. |
| 34 | # also _do_rsync uses retry for waiting wnen resource will be unlocked |
| 35 | # TODO: check for url compatibility (local->remote, remote->local, |
| 36 | # local->local) |
| 37 | # TODO: push method - upstream mirrors |
| 38 | dest = self.root.urljoin(dest) |
| 39 | allextra = self.rsync_extra_params |
| 40 | if extra is not None: |
| 41 | allextra = ' '.join((allextra, extra)) |
| 42 | cmd = 'rsync {opts} {allextra} {source} {dest}'.format(**(locals())) |
| 43 | return self.shell.shell(cmd)[1] |
| 44 | |
| 45 | def _rsync_ls(self, dirname=None, pattern=r'.*', opts=''): |
| 46 | extra = '--no-v' |
| 47 | out = self._do_rsync(dest=dirname, opts=opts, extra=extra) |
| 48 | regexp = re.compile(pattern) |
| 49 | out = [_ for _ in out.splitlines() |
| 50 | if (_.split()[-1] != '.') and |
| 51 | (regexp.match(_.split()[-1]) is not None)] |
| 52 | return out |
| 53 | |
| 54 | def ls(self, dirname=None, pattern=r'.*'): |
| 55 | self.logger.debug('ls on "{}", pattern="{}"'.format(dirname, pattern)) |
| 56 | out = self._rsync_ls(dirname, pattern=pattern) |
| 57 | out = [_.split()[-1] for _ in out] |
| 58 | return out |
| 59 | |
| 60 | def ls_dirs(self, dirname=None, pattern=r'.*'): |
| 61 | self.logger.debug('ls dirs on "{}", pattern="{}"' |
| 62 | ''.format(dirname, pattern)) |
| 63 | out = self._rsync_ls(dirname, pattern=pattern) |
| 64 | out = [_.split()[-1] for _ in out if _.startswith('d')] |
| 65 | return out |
| 66 | |
| 67 | def ls_symlinks(self, dirname=None, pattern=r'.*'): |
| 68 | self.logger.debug('ls symlinks on "{}", pattern="{}"' |
| 69 | ''.format(dirname, pattern)) |
| 70 | out = self._rsync_ls(dirname, pattern=pattern, opts='-l') |
| 71 | out = [_.split()[-3:] for _ in out if _.startswith('l')] |
| 72 | out = [[_[0], _[-1]] for _ in out] |
| 73 | return out |
| 74 | |
| 75 | def rmfile(self, filename): |
| 76 | '''Removes file on rsync_url.''' |
| 77 | report_name = filename |
| 78 | dirname, filename = os.path.split(filename) |
| 79 | dirname = self.root.dirname(dirname) |
| 80 | source = self.root.dirname(self.tmp.empty_dir) |
| 81 | opts = "-r --delete --include={} '--exclude=*'".format(filename) |
| 82 | self.logger.info('Removing file "{}"'.format(report_name)) |
| 83 | return self._do_rsync(source=source, dest=dirname, opts=opts) |
| 84 | |
| 85 | def cleandir(self, dirname): |
| 86 | '''Removes directories (recursive) on rsync_url''' |
| 87 | dirname = self.root.dirname(dirname) |
| 88 | source = self.root.dirname(self.tmp.empty_dir) |
| 89 | opts = "-a --delete" |
| 90 | self.logger.info('Cleaning directory "{}"'.format(dirname)) |
| 91 | return self._do_rsync(source=source, dest=dirname, opts=opts) |
| 92 | |
| 93 | def rmdir(self, dirname): |
| 94 | '''Removes directories (recursive) on rsync_url''' |
| 95 | self.logger.info('Removing directory "{}"'.format(dirname)) |
| 96 | self.cleandir(dirname) |
| 97 | return self.rmfile(self.root.filename(dirname)) |
| 98 | |
| 99 | def mkdir(self, dirname): |
| 100 | '''Creates directories (recirsive, like mkdir -p) on rsync_url''' |
| 101 | source = self.root.dirname(self.tmp.get_temp_dir(dirname)) |
| 102 | opts = "-a" |
| 103 | self.logger.info('Creating directory "{}"'.format(dirname)) |
| 104 | return self._do_rsync(source=source, opts=opts) |
| 105 | |
| 106 | def symlink(self, symlink, target): |
| 107 | '''Creates symlink targeted to target''' |
| 108 | source = self.tmp.get_symlink_to(target) |
| 109 | symlink = self.root.filename(symlink) |
| 110 | opts = "-l" |
| 111 | self.rmfile(symlink) |
| 112 | self.logger.info('Creating symlink "{}" -> "{}"' |
| 113 | ''.format(symlink, target)) |
| 114 | return self._do_rsync(source=source, dest=symlink, opts=opts) |
| 115 | |
| 116 | def push(self, source, dest=None, extra=None): |
| 117 | '''Push source to destination''' |
| 118 | opts = '--archive --force --ignore-errors --delete' |
| 119 | self.logger.info('Push "{}" to "{}"'.format(source, dest)) |
| 120 | return self._do_rsync(source=source, dest=dest, opts=opts, extra=extra) |