Merge "Optimize kitchen tests for Travis CI"
diff --git a/README.rst b/README.rst
index 300c3e5..6321347 100644
--- a/README.rst
+++ b/README.rst
@@ -66,6 +66,20 @@
Version: GnuPG v1
...
+Define s3 endpoint:
+
+.. code-block:: yaml
+
+ parameters:
+ aptly:
+ server:
+ endpoint:
+ mys3endpoint:
+ engine: s3
+ awsAccessKeyID: xxxx
+ awsSecretAccessKey: xxxx
+ bucket: test
+
Example pillar
==============
diff --git a/aptly/files/aptly.conf b/aptly/files/aptly.conf
index 71c206f..0f891f9 100644
--- a/aptly/files/aptly.conf
+++ b/aptly/files/aptly.conf
@@ -1,20 +1,47 @@
{%- from "aptly/map.jinja" import server with context %}
{
- "rootDir": "{{ server.root_dir }}",
+ "rootDir": "{{ server.get('config', {}).root_dir|default(server.root_dir) }}",
"downloadConcurrency": 4,
- "architectures": [ "amd64" ],
- "dependencyFollowSuggests": false,
- "dependencyFollowRecommends": false,
- "dependencyFollowAllVariants": false,
- "dependencyFollowSource": false,
+ "architectures": [ {% for arch in server.architectures|default([]) %}"{{ arch }}"{% if not loop.last %},{% endif %}{% endfor %} ],
+ "dependencyFollowSuggests": {{ server.dependency.follow.suggests|lower }},
+ "dependencyFollowRecommends": {{ server.dependency.follow.recommends|lower }},
+ "dependencyFollowAllVariants": {{ server.dependency.follow.all_variants|lower }},
+ "dependencyFollowSource": {{ server.dependency.follow.source|lower }},
+ {%- if server.endpoint is defined %}
+ "S3PublishEndpoints": {
+ {%- for name, endpoint in server.endpoint.iteritems() %}
+ {%- if endpoint.engine == 's3' %}
+ "{{ name }}": {
+ {%- for key, value in endpoint.iteritems() %}
+ "{{ key }}": "{{ value }}"{% if not loop.last %},{% endif %}
+ {%- endfor %}
+ }
+ {%- endif %}
+ {%- endfor %}
+ },
+ "SwiftPublishEndpoints": {
+ {%- for name, endpoint in server.endpoint.iteritems() %}
+ {%- if endpoint.engine == 'swift' %}
+ "{{ name }}": {
+ {%- for key, value in endpoint.iteritems() %}
+ "{{ key }}": "{{ value }}"{% if not loop.last %},{% endif %}
+ {%- endfor %}
+ }
+ {%- endif %}
+ {%- endfor %}
+ },
+ {%- endif %}
{%- if server.secure %}
- "gpgDisableSign": false,
- "gpgDisableVerify": false,
+ "gpgDisableSign": {{ server.gpg.disable_sign|default(False)|lower }},
+ "gpgDisableVerify": {{ server.gpg.disable_verify|default(False)|lower }},
{%- else %}
- "gpgDisableSign": true,
- "gpgDisableVerify": true,
+ "gpgDisableSign": {{ server.gpg.disable_sign|default(True)|lower }},
+ "gpgDisableVerify": {{ server.gpg.disable_verify|default(True)|lower }},
{%- endif %}
"downloadSourcePackages": false,
"ppaDistributorID": "ubuntu",
"ppaCodename": ""
}
+{#-
+vim: syntax=jinja
+-#}
diff --git a/aptly/files/aptly_mirror_update.sh b/aptly/files/aptly_mirror_update.sh
index e61e7a4..3deb8ac 100755
--- a/aptly/files/aptly_mirror_update.sh
+++ b/aptly/files/aptly_mirror_update.sh
@@ -1,71 +1,90 @@
#!/bin/bash
-
+try=0
+retval=666
SCRIPT=$(basename $0)
MAXTRIES=3
VERBOSE=0
SNAPSHOT=0
+MIRROR=""
log_info() {
- logger -p user.info -t ${SCRIPT} "$*"
- [ $VERBOSE -eq 1 ] && echo "[INFO] $*"
+ logger -p user.info -t ${SCRIPT} "$*"
+ [ $VERBOSE -eq 1 ] && echo "[INFO] $*"
}
log_error() {
- logger -p user.error -t ${SCRIPT} "$*"
- echo "[ERROR] $*" >&2
+ logger -p user.error -t ${SCRIPT} "$*"
+ echo "[ERROR] $*" >&2
}
-if [[ "$*" == *--verbose* || "$*" == *-v* ]]; then
- VERBOSE=1
-fi
+update_mirror() {
+ if [[ "$1" == "" ]]; then
+ log_error "Mirror has to be specified."
+ exit 3
+ fi
+ while [[ $retval -ne 0 && $try -lt $MAXTRIES ]]; do
+ log_info "Starting update of mirror ${1}"
+ if [[ $VERBOSE -eq 0 ]]; then
+ out=$(aptly mirror update -force=true ${1} >/dev/null 2>&1)
+ else
+ aptly mirror update -force=true ${1}
+ fi
-if [[ "$*" == *--snapshot* || "$*" == *-s* ]]; then
- SNAPSHOT=1
-fi
+ retval=$?
-MIRRORS=$(aptly mirror list --raw 2>&1)
-if [[ $? -ne 0 ]]; then
- log_error "$MIRRORS"
- exit 1
-fi
+ if [[ $retval -ne 0 ]]; then
+ try=$[ $try + 1 ]
+ log_error "Failed to update mirror ${1}, try=${try}"
+ [ ! -z "$out" ] && log_error "${out}"
+ else
+ log_info "Synced mirror ${1}"
+ if [[ $SNAPSHOT -eq 1 ]]; then
+ snapshot_name="${1}-$(date +%s)"
+ if [[ $VERBOSE -eq 0 ]]; then
+ out=$(aptly snapshot create ${snapshot_name} from mirror ${1} >/dev/null 2>&1)
+ else
+ aptly snapshot create ${snapshot_name} from mirror ${1}
+ fi
-for mirror in $MIRRORS; do
- try=0
- retval=666
+ retval=$?
+ if [[ $retval -ne 0 ]]; then
+ log_error "Failed to create snapshot ${snapshot_name} from mirror ${1}"
+ [ ! -z "$out" ] && log_error "$out"
+ else
+ log_info "Created snapshot ${snapshot_name} from mirror ${1}"
+ fi
+ fi
+ break
+ fi
+ done
+ try=0
+ retval=666
+}
- while [[ $retval -ne 0 && $try -lt $MAXTRIES ]]; do
- log_info "Starting update of mirror ${mirror}"
- if [[ $VERBOSE -eq 0 ]]; then
- out=$(aptly mirror update -force=true ${mirror} >/dev/null 2>&1)
- else
- aptly mirror update -force=true ${mirror}
- fi
-
- retval=$?
-
- if [[ $retval -ne 0 ]]; then
- try=$[ $try + 1 ]
- log_error "Failed to update mirror ${mirror}, try=${try}"
- [ ! -z "$out" ] && log_error "${out}"
- else
- log_info "Synced mirror ${mirror}"
- if [[ $SNAPSHOT -eq 1 ]]; then
- snapshot_name="${mirror}-$(date +%s)"
- if [[ $VERBOSE -eq 0 ]]; then
- out=$(aptly snapshot create ${snapshot_name} from mirror ${mirror} >/dev/null 2>&1)
- else
- aptly snapshot create ${snapshot_name} from mirror ${mirror}
- fi
-
- retval=$?
- if [[ $retval -ne 0 ]]; then
- log_error "Failed to create snapshot ${snapshot_name} from mirror ${mirror}"
- [ ! -z "$out" ] && log_error "$out"
- else
- log_info "Created snapshot ${snapshot_name} from mirror ${mirror}"
- fi
- fi
- break
- fi
- done
+while getopts "m:s?v?" option
+do
+ case "${option}"
+ in
+ m|\?) MIRROR=${OPTARG};;
+ s|\?) SNAPSHOT=1;;
+ v|\?) VERBOSE=1;;
+ esac
done
+
+if [[ "$MIRROR" != "" ]]; then
+ MIRROR_DETAIL=$(aptly mirror show ${MIRROR})
+ if [[ $? -ne 0 ]]; then
+ log_error "$MIRROR_DETAIL"
+ exit 1
+ fi
+ update_mirror "${MIRROR}"
+else
+ MIRRORS=$(aptly mirror list --raw 2>&1)
+ if [[ $? -ne 0 ]]; then
+ log_error "$MIRRORS"
+ exit 1
+ fi
+ for mirror in $MIRRORS; do
+ update_mirror "${mirror}"
+ done
+fi
diff --git a/aptly/files/aptly_publish_update.sh b/aptly/files/aptly_publish_update.sh
new file mode 100644
index 0000000..cbc8ca5
--- /dev/null
+++ b/aptly/files/aptly_publish_update.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+CLEANUP=0
+VERBOSE=0
+START_API=0
+RECREATE=0
+FORCE_OVERWRITE=0
+PUBLISHER_OPTIONS=""
+
+log_info() {
+ logger -p user.info -t ${SCRIPT} "$*"
+ [ $VERBOSE -eq 1 ] && echo "[INFO] $*"
+}
+
+log_error() {
+ logger -p user.error -t ${SCRIPT} "$*"
+ echo "[ERROR] $*" >&2
+}
+
+at_exit() {
+ pgrep -f "aptly api serve" | xargs kill -15
+}
+trap at_exit EXIT
+
+while getopts "a?c?f?r?v?" option
+do
+ case "${option}"
+ in
+ a|\?) START_API=1;;
+ c|\?) CLEANUP=1;;
+ f|\?) FORCE_OVERWRITE=1;;
+ r|\?) RECREATE=1;;
+ v|\?) VERBOSE=1;;
+ esac
+done
+
+if [[ $START_API -eq 1 ]]; then
+ nohup aptly api serve --no-lock > /dev/null 2>&1 </dev/null &
+fi
+if [[ $RECREATE -eq 1 ]]; then
+ PUBLISHER_OPTIONS+=" --recreate"
+fi
+if [[ $FORCE_OVERWRITE -eq 1 ]]; then
+ PUBLISHER_OPTIONS+=" --force-overwrite"
+fi
+
+aptly-publisher --timeout=1200 publish -v -c /etc/aptly-publisher.yaml --url http://127.0.0.1:8080 --architectures amd64 $PUBLISHER_OPTIONS
+
+if [[ $? -ne 0 ]]; then
+ echo "Aptly Publisher failed."
+ exit 1
+fi
+
+if [[ $CLEANUP -eq 1 ]]; then
+ SNAPSHOT_LIST="$(aptly snapshot list --raw)"
+ if [[ "$SNAPSHOT_LIST" != "" ]]; then
+ log_info "Deleting unpublished snapshots"
+ echo $SNAPSHOT_LIST | grep -E '*' | xargs -n 1 aptly snapshot drop
+ fi
+ log_info "Cleaning Aptly DB"
+ aptly db cleanup
+fi
+exit 0
\ No newline at end of file
diff --git a/aptly/map.jinja b/aptly/map.jinja
index e0d4853..5538f99 100644
--- a/aptly/map.jinja
+++ b/aptly/map.jinja
@@ -1,10 +1,18 @@
{%- set server = salt['grains.filter_by']({
- 'Debian': {
+ 'default': {
'source': {
'engine': 'pkg',
'pkgs': ['aptly', 'bzip2', 'cron'],
},
+ 'dependency': {
+ 'follow': {
+ 'suggests': false,
+ 'recommends': false,
+ 'all_variants': false,
+ 'source': false,
+ }
+ },
'home_dir': '/var/lib/aptly',
'root_dir': '/srv/aptly',
'secure': True,
diff --git a/aptly/publisher.sls b/aptly/publisher.sls
index 0ddb9ee..f11dcfd 100644
--- a/aptly/publisher.sls
+++ b/aptly/publisher.sls
@@ -22,25 +22,23 @@
{%- elif publisher.source.engine == 'docker' %}
{% set publisherImage = (publisher.source.image|default('tcpcloud/aptly-publisher')).split(':') %}
+{% set publisherImageTag = publisherImage[1] if publisherImage|length > 1 else 'latest' %}
{% set registry = publisher.source.registry + "/" if publisher.source.registry is defined else "" %}
{{ publisherImage[0] }}:
dockerng.image_present:
- - name: {{ registry }}{{ publisherImage[0] }}
-{%- if publisherImage|length > 1 %}
- tag: {{ publisherImage[1] }}
-{%- else %}
- tag: latest
-{%- endif %}
+ - name: {{ registry }}{{ publisherImage[0] }}:{{ publisherImageTag}}
force: true
+ - require_in:
+ - file: publisher_wrapper
publisher_wrapper:
file.managed:
- name: /usr/local/bin/aptly-publisher
- source: salt://aptly/files/aptly-publisher
- - template: jinja
- defaults:
- image: {{ publisher.source.image|default('tcpcloud/aptly-publisher') }}
+ image: {{ registry }}{{ publisherImage[0] }}:{{ publisherImageTag}}
+ - template: jinja
- mode: 755
publisher_installed:
@@ -51,6 +49,14 @@
{%- endif %}
+aptly_publish_script:
+ file.managed:
+ - name: /usr/local/bin/aptly_publish_update.sh
+ - source: salt://aptly/files/aptly_publish_update.sh
+ - user: root
+ - group: root
+ - mode: 755
+
publisher_yaml:
file.managed:
- name: /etc/aptly-publisher.yaml
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 29fb975..4ee4e0f 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -124,7 +124,7 @@
}
salt_run() {
- [ -e ${VEN_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
+ [ -e ${VENV_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
salt-call ${SALT_OPTS} $*
}