Add ability to keep local cache for debmirror's repo syncs
Add new cache_dir parameter to keep local cache for debmirror
repo and re-use it. During first run debmirror will create
cache and then using hardlinks cache will be copied to target
repo. All next runs will copy cache first and then run debmirror
with prepared cached.
Change-Id: Ib7932e64c75743b06b3b12507fe87a30d4d93311
(cherry picked from commit 964ba9a2b45335c2c7cc2bcf7aab094296de0d36)
diff --git a/README.rst b/README.rst
index 4c1b825..dde2942 100644
--- a/README.rst
+++ b/README.rst
@@ -30,6 +30,7 @@
arch: [ 'amd64' ]
mirror_host: "mirror.mirantis.com" # rsync
mirror_root: ':mirror/nightly/ubuntu/'
+ cache_dir: "/var/www/mirror/.cache/ubuntu"
target_dir: "/var/www/mirror/ubuntu/"
log_file: "/var/www/mirror/target01_log.log"
dist: [ xenial ] #, xenial-security, xenial-updates ]
@@ -47,6 +48,9 @@
08: "--include='/main(.*)python-(.*)doc'"
09: "--include='/main(.*)python-(.*)network'"
+Parameter cache_dir is optional and can be used to avoid extra disk space
+usage for repos, which can have same packages, by using hardlinks to files.
+
**Documentation and bugs**
* http://salt-formulas.readthedocs.io/
@@ -71,3 +75,4 @@
* #salt-formulas @ irc.freenode.net
Use this IRC channel in case of any questions or feedback which is always
welcome
+
diff --git a/_states/debmirror.py b/_states/debmirror.py
index 83c51a6..2c0bb8a 100644
--- a/_states/debmirror.py
+++ b/_states/debmirror.py
@@ -101,6 +101,11 @@
return env
+def _check_mirror_cache(cache_dir):
+ cacheReadinessKeyFile = os.path.join(cache_dir, '.ready')
+ return os.path.isfile(cacheReadinessKeyFile)
+
+
def _get_cmdline(name, tgt):
cmdline = " debmirror "
if tgt.get('extra_flags'):
@@ -129,7 +134,15 @@
for key, value in enumerate(sorted(tgt['filter'])):
cmdline += " " + tgt['filter'][value]
if tgt.get('target_dir', False):
- cmdline += ' ' + _get_target_path(name, tgt)['target_dir']
+ target_dir = _get_target_path(name, tgt)['target_dir']
+ if tgt.get('cache_dir'):
+ cache_dir = tgt.get('cache_dir')
+ if _check_mirror_cache(cache_dir):
+ cmdline += ' ' + target_dir
+ else:
+ cmdline += ' ' + cache_dir
+ else:
+ cmdline += ' ' + target_dir
return cmdline
@@ -154,8 +167,29 @@
fh.setFormatter(fh_format)
log2file = logging.getLogger("debmirror")
log2file.addHandler(fh)
- result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
- env=env_vars)
+ # check cache usage
+ if tgt.get('cache_dir'):
+ cache_dir = tgt.get('cache_dir')
+ target_dir = _get_target_path(name, tgt)['target_dir']
+ cpline = "find . -name *.deb -exec cp -rlf --parent {} %s \\;" % (
+ target_dir)
+ if _check_mirror_cache(cache_dir):
+ result = __salt__['cmd.run_all'](cpline, redirect_stderr=True,
+ cwd=cache_dir, env=env_vars)
+ if result['retcode'] == 0:
+ log2file.debug(result['stdout'])
+ result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
+ env=env_vars)
+ else:
+ result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
+ env=env_vars)
+ if result['retcode'] == 0:
+ log2file.debug(result['stdout'])
+ result = __salt__['cmd.run_all'](cpline, redirect_stderr=True,
+ cwd=cache_dir, env=env_vars)
+ else:
+ result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
+ env=env_vars)
log2file.debug(result['stdout'])
# destroy file logger
for i in list(log2file.handlers):
diff --git a/debmirror/client/init.sls b/debmirror/client/init.sls
index 7e7e5f0..0230923 100644
--- a/debmirror/client/init.sls
+++ b/debmirror/client/init.sls
@@ -13,6 +13,10 @@
- name: {{ mirror_name }}
- require:
- debmirror_client_packages
+ {%- if opts.get('cache_dir', '') != '' %}
+ - debmirror_cache_dir
+ - debmirror_target_dir
+ {%- endif %}
{%- if grains['saltversioninfo'][0] >= 2017 and grains['saltversioninfo'][1] >= 7 %}
- retry:
attempts: {{ opts.get('fetch_retry' , 1) }}
@@ -20,8 +24,31 @@
interval: 5
splay: 2
{%- endif %}
+
+{%- if opts.get('cache_dir', '') != '' %}
+debmirror_cache_dir:
+ file.directory:
+ - name: '{{ opts.get('cache_dir') }}'
+ - user: root
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+debmirror_target_dir:
+ file.directory:
+ - name: '{{ opts.get('target_dir') }}'
+ - user: root
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+debmirror_cache_readiness:
+ file.managed:
+ - name: '{{ opts.get('cache_dir') }}/.ready'
+ - replace: False
+ - require:
+ - debmirror_{{ mirror_name }}_present
+{%- endif %}
+
{% endif %}
{%- endfor %}
{% endif %}
-
diff --git a/debmirror/schemas/client.yaml b/debmirror/schemas/client.yaml
index da7debc..65a7914 100644
--- a/debmirror/schemas/client.yaml
+++ b/debmirror/schemas/client.yaml
@@ -93,6 +93,10 @@
type: string
example: "/var/www/mirror/ubuntu/"
description: "Destination folder for mirror"
+ cache_dir:
+ type: string
+ example: "/var/www/mirror/.snapshots/ubuntu"
+ description: "Destination folder for keeping local cache of mirror"
log_file:
type: string
example: "/var/www/mirror/target01_log.log"
diff --git a/tests/pillar/client.sls b/tests/pillar/client.sls
index 005dd04..6a8fd98 100644
--- a/tests/pillar/client.sls
+++ b/tests/pillar/client.sls
@@ -15,6 +15,7 @@
dist: [ xenial ] #, xenial-security, xenial-updates ]
section: [ main ] #, multiverse, restricted, universe ]
exclude_deb_section: [ 'games', gnome, Xfce, sound, electronics, graphics, hamradio , doc, localization, kde, video ]
+ cache_dir: "/tmp/mirror/.cache/ubuntu"
filter:
00: "--exclude='/*'" # exclude all for test..
01: "--include='/vim-tiny'" # and include something small.