Add ability to keep local cache for debmirror's repo syncs
Add new cache_dir parameter to keep local cache for debmirror
repo and re-use it. During first run debmirror will create
cache and then using hardlinks cache will be copied to target
repo. All next runs will copy cache first and then run debmirror
with prepared cached.
Change-Id: Ib7932e64c75743b06b3b12507fe87a30d4d93311
diff --git a/README.rst b/README.rst
index b89a4cc..653a21d 100644
--- a/README.rst
+++ b/README.rst
@@ -30,6 +30,7 @@
arch: [ 'amd64' ]
mirror_host: "mirror.mirantis.com" # rsync
mirror_root: ':mirror/nightly/ubuntu/'
+ cache_dir: "/var/www/mirror/.cache/ubuntu"
target_dir: "/var/www/mirror/ubuntu/"
log_file: "/var/www/mirror/target01_log.log"
dist: [ xenial ] #, xenial-security, xenial-updates ]
@@ -46,3 +47,7 @@
07: "--include='/main(.*)manpages'"
08: "--include='/main(.*)python-(.*)doc'"
09: "--include='/main(.*)python-(.*)network'"
+
+Parameter cache_dir is optional and can be used to avoid extra disk space
+usage for repos, which can have same packages, by using hardlinks to files.
+
diff --git a/_states/debmirror.py b/_states/debmirror.py
index 83c51a6..2c0bb8a 100644
--- a/_states/debmirror.py
+++ b/_states/debmirror.py
@@ -101,6 +101,11 @@
return env
+def _check_mirror_cache(cache_dir):
+ cacheReadinessKeyFile = os.path.join(cache_dir, '.ready')
+ return os.path.isfile(cacheReadinessKeyFile)
+
+
def _get_cmdline(name, tgt):
cmdline = " debmirror "
if tgt.get('extra_flags'):
@@ -129,7 +134,15 @@
for key, value in enumerate(sorted(tgt['filter'])):
cmdline += " " + tgt['filter'][value]
if tgt.get('target_dir', False):
- cmdline += ' ' + _get_target_path(name, tgt)['target_dir']
+ target_dir = _get_target_path(name, tgt)['target_dir']
+ if tgt.get('cache_dir'):
+ cache_dir = tgt.get('cache_dir')
+ if _check_mirror_cache(cache_dir):
+ cmdline += ' ' + target_dir
+ else:
+ cmdline += ' ' + cache_dir
+ else:
+ cmdline += ' ' + target_dir
return cmdline
@@ -154,8 +167,29 @@
fh.setFormatter(fh_format)
log2file = logging.getLogger("debmirror")
log2file.addHandler(fh)
- result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
- env=env_vars)
+ # check cache usage
+ if tgt.get('cache_dir'):
+ cache_dir = tgt.get('cache_dir')
+ target_dir = _get_target_path(name, tgt)['target_dir']
+ cpline = "find . -name *.deb -exec cp -rlf --parent {} %s \\;" % (
+ target_dir)
+ if _check_mirror_cache(cache_dir):
+ result = __salt__['cmd.run_all'](cpline, redirect_stderr=True,
+ cwd=cache_dir, env=env_vars)
+ if result['retcode'] == 0:
+ log2file.debug(result['stdout'])
+ result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
+ env=env_vars)
+ else:
+ result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
+ env=env_vars)
+ if result['retcode'] == 0:
+ log2file.debug(result['stdout'])
+ result = __salt__['cmd.run_all'](cpline, redirect_stderr=True,
+ cwd=cache_dir, env=env_vars)
+ else:
+ result = __salt__['cmd.run_all'](cmdline, redirect_stderr=True,
+ env=env_vars)
log2file.debug(result['stdout'])
# destroy file logger
for i in list(log2file.handlers):
diff --git a/debmirror/client/init.sls b/debmirror/client/init.sls
index 7e7e5f0..0230923 100644
--- a/debmirror/client/init.sls
+++ b/debmirror/client/init.sls
@@ -13,6 +13,10 @@
- name: {{ mirror_name }}
- require:
- debmirror_client_packages
+ {%- if opts.get('cache_dir', '') != '' %}
+ - debmirror_cache_dir
+ - debmirror_target_dir
+ {%- endif %}
{%- if grains['saltversioninfo'][0] >= 2017 and grains['saltversioninfo'][1] >= 7 %}
- retry:
attempts: {{ opts.get('fetch_retry' , 1) }}
@@ -20,8 +24,31 @@
interval: 5
splay: 2
{%- endif %}
+
+{%- if opts.get('cache_dir', '') != '' %}
+debmirror_cache_dir:
+ file.directory:
+ - name: '{{ opts.get('cache_dir') }}'
+ - user: root
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+debmirror_target_dir:
+ file.directory:
+ - name: '{{ opts.get('target_dir') }}'
+ - user: root
+ - group: root
+ - dir_mode: 755
+ - makedirs: True
+debmirror_cache_readiness:
+ file.managed:
+ - name: '{{ opts.get('cache_dir') }}/.ready'
+ - replace: False
+ - require:
+ - debmirror_{{ mirror_name }}_present
+{%- endif %}
+
{% endif %}
{%- endfor %}
{% endif %}
-
diff --git a/debmirror/schemas/client.yaml b/debmirror/schemas/client.yaml
index da7debc..65a7914 100644
--- a/debmirror/schemas/client.yaml
+++ b/debmirror/schemas/client.yaml
@@ -93,6 +93,10 @@
type: string
example: "/var/www/mirror/ubuntu/"
description: "Destination folder for mirror"
+ cache_dir:
+ type: string
+ example: "/var/www/mirror/.snapshots/ubuntu"
+ description: "Destination folder for keeping local cache of mirror"
log_file:
type: string
example: "/var/www/mirror/target01_log.log"
diff --git a/tests/pillar/client.sls b/tests/pillar/client.sls
index 005dd04..6a8fd98 100644
--- a/tests/pillar/client.sls
+++ b/tests/pillar/client.sls
@@ -15,6 +15,7 @@
dist: [ xenial ] #, xenial-security, xenial-updates ]
section: [ main ] #, multiverse, restricted, universe ]
exclude_deb_section: [ 'games', gnome, Xfce, sound, electronics, graphics, hamradio , doc, localization, kde, video ]
+ cache_dir: "/tmp/mirror/.cache/ubuntu"
filter:
00: "--exclude='/*'" # exclude all for test..
01: "--include='/vim-tiny'" # and include something small.