Change image build logic
* Implement pre-build.sh
Due our CI strategy, its kinda hard to fetch all resources
during docker build stage.
split logic to:
pre-build.sh - script, will be ran by jenkins,
with ssh-agent wrapper
extra/build.sh - script, will be ran during docker build
* Merge RUN into one step, to reduce layer amount
* Fix 'tempest' dependency - always isntall from local
copy
Revert "Add authorization for gerrit repositories"
This reverts commit 920ac7ce0c2b9a67bdc9186cd5e75b0b2b93ed22.
Change-Id: I2edf0c6e337eef21b8f8f1d6c2f1636ccde60932
(cherry picked from commit b2199a49245a7b256ab68a7974f964fbdcaf2ec4)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8fce603
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+data/
diff --git a/Dockerfile b/Dockerfile
index b3095e0..bcc2ba8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,26 +1,26 @@
FROM ubuntu:16.04
LABEL maintainer="qa@mirantis.com"
-ARG TEMPEST_TAG="mcp/pike"
-# Tempest plugins
-ARG HORIZON_TAG="mcp/pike"
-ARG BARBICAN_TAG="mcp/pike"
-ARG DESIGNATE_TAG="mcp/pike"
-ARG HEAT_TAG="mcp/pike"
-ARG IRONIC_TAG="mcp/pike"
-ARG MANILA_TAG="mcp/pike"
-ARG TELEMETRY_TAG="mcp/pike"
-# global openstack requirements repo
-# This is the commit where tempest version is 17.2.0 in u-c
-ARG UPPER_CONSTRAINTS_TAG="83b1de74834fbcdbe749e270c29e76cdc28f85d5"
-
ENV DEBIAN_FRONTEND=noninteractive \
DEBCONF_NONINTERACTIVE_SEEN=true \
LANG=C.UTF-8 \
- LANGUAGE=$LANG
-SHELL ["/bin/bash", "-xec"]
+ LANGUAGE=$LANG \
+ LOG_DIR='/root/tempest'
+SHELL ["/bin/bash", "-o", "pipefail", "-xec"]
USER root
-# pre-update
+ADD data /var/lib/
+
+COPY extra/build.sh /
+COPY bin/entrypoint.sh /usr/bin/
+COPY bin/run_tempest.sh /usr/bin/run-tempest
+COPY bin/skiplists /var/lib/tempest/skiplists
+
+# Patch junitxml library to include skipped tests in the test report totals header
+COPY bin/patches/junitxml_init.patch /var/lib/junitxml_init.patch
+
+# Patch os_testr library to fix issue ralated with tempest black-list option
+COPY bin/patches/regex_builder.patch /var/lib/regex_builder.patch
+
RUN cd /etc/apt/ && echo > sources.list \
&& echo "deb [arch=amd64] http://cz.archive.ubuntu.com/ubuntu xenial main restricted universe multiverse" >> sources.list \
&& echo "deb [arch=amd64] http://cz.archive.ubuntu.com/ubuntu xenial-updates main restricted universe multiverse" >> sources.list \
@@ -29,86 +29,27 @@
&& echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > apt.conf.d/docker-gzip-indexes \
&& echo 'APT::Get::Install-Recommends "false"; APT::Get::Install-Suggests "false";' > apt.conf.d/docker-recommends \
&& apt-get update && apt-get upgrade -y \
- && apt-get install -y vim-tiny git-core python-pip libffi-dev libssl-dev python-dev python3-dev iputils-ping patch
-
+ && apt-get install -y vim-tiny git-core python-pip libffi-dev libssl-dev python-dev python3-dev iputils-ping patch \
#Due to upstream bug we should use fixed version of pip
-RUN pip install -U 'pip==9.0.3'
-WORKDIR /var/lib/
-# TODO migrate to downstrem mirror?
-RUN git clone https://github.com/openstack/requirements.git openstack_requirements && \
- pushd openstack_requirements; git checkout $UPPER_CONSTRAINTS_TAG; popd ;
-
-RUN sed -i -e 's|tempest===17.2.0|git+https://gerrit.mcp.mirantis.net/packaging/sources/tempest@mcp/pike#egg=tempest|g' \
- /var/lib/openstack_requirements/upper-constraints.txt;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/tempest-horizon tempest-horizon-plugin ; \
- pushd tempest-horizon-plugin ; git checkout $HORIZON_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/barbican-tempest-plugin && \
- pushd barbican-tempest-plugin; git checkout $BARBICAN_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/designate-tempest-plugin && \
- pushd designate-tempest-plugin; git checkout $DESIGNATE_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/heat-tempest-plugin && \
- pushd heat-tempest-plugin; git checkout $HEAT_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/ironic-tempest-plugin && \
- pushd ironic-tempest-plugin; git checkout $IRONIC_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/manila-tempest-plugin && \
- pushd manila-tempest-plugin; git checkout $MANILA_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/telemetry-tempest-plugin && \
- pushd telemetry-tempest-plugin; git checkout $TELEMETRY_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt . ; \
- popd;
-
-RUN git clone https://gerrit.mcp.mirantis.net/packaging/sources/tempest && \
- pushd tempest; git checkout $TEMPEST_TAG; \
- pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt ; \
- pip install . ; popd;
-
-RUN pip install junitxml
-# Patch junitxml library to include skipped tests in the test report totals header
-COPY bin/patches/junitxml_init.patch /tmp/junitxml_init.patch
-RUN patch /usr/local/lib/python2.7/dist-packages/junitxml/__init__.py /tmp/junitxml_init.patch
-
-# Patch os_testr library to fix issue ralated with tempest black-list option
-COPY bin/patches/regex_builder.patch /tmp/regex_builder.patch
-RUN patch /usr/local/lib/python2.7/dist-packages/os_testr/regex_builder.py /tmp/regex_builder.patch
-
-# Cleanup.
-RUN apt-get -y purge libx11-data xauth libxmuu1 libxcb1 libx11-6 libxext6 \
- ppp pppconfig pppoeconf popularity-contest cpp gcc g++ libssl-doc
-RUN apt-get -y autoremove; apt-get -y clean;
-RUN rm -rf /root/.cache
-RUN rm -rf /var/lib/apt/lists/*
-
-RUN echo "BUILD_TIMESTAMP=$(date '+%Y-%m-%d-%H-%M-%S' -u)" > /etc/build_timestamp
+ && pip install -U 'pip==18.0' \
+# Do main stuff
+ && /build.sh buildme \
+# Cleanup
+ && apt-get -y purge libx11-data xauth libxmuu1 libxcb1 libx11-6 libxext6 \
+ ppp pppconfig pppoeconf popularity-contest cpp gcc g++ libssl-doc \
+ && apt-get -y autoremove; apt-get -y clean \
+ && rm -rf /root/.cache \
+ && rm -rf /var/lib/apt/lists/*
#
COPY bin/entrypoint.sh /usr/bin/entrypoint.sh
COPY bin/run_tempest.sh /usr/bin/run-tempest
-ENV LOG_DIR='/root/tempest'
-
-COPY bin/skiplists /var/lib/tempest/skiplists
+RUN patch /usr/local/lib/python2.7/dist-packages/junitxml/__init__.py /var/lib/junitxml_init.patch
+RUN patch /usr/local/lib/python2.7/dist-packages/os_testr/regex_builder.py /var/lib/regex_builder.patch
WORKDIR /var/lib/tempest/
ENTRYPOINT ["/usr/bin/entrypoint.sh"]
# Build
-# docker build -t docker-ci-tempest:$(date "+%Y_%m_%d_%H_%M_%S").
+# docker build --no-cache -t docker-ci-tempest:$(date "+%Y_%m_%d_%H_%M_%S") .
diff --git a/extra/build.sh b/extra/build.sh
new file mode 120000
index 0000000..ae5c29e
--- /dev/null
+++ b/extra/build.sh
@@ -0,0 +1 @@
+pre-build.sh
\ No newline at end of file
diff --git a/extra/pre-build.sh b/extra/pre-build.sh
new file mode 100755
index 0000000..ed1748b
--- /dev/null
+++ b/extra/pre-build.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+
+set -ex
+
+export GERRIT_NAME=${GERRIT_NAME:-mcp-jenkins}
+# Would delete all .git folders.
+# Keep in mind, you would not able to get any other version or history!
+# Will ignore 'tempest' itself bz of:
+# ```
+# Exception: Versioning for this project requires either an sdist tarball, or access to an upstream git repository.
+# It's also possible that there is a mismatch between the package name in setup.cfg and the argument given to pbr.version.VersionInfo.
+# Project name tempest was given, but was not able to be found.
+# ```
+# FIXME|TODO Disabled by default.
+# Currently, pip not able to install any package w\o .git directory.
+# Should it be re-writed to tgz install or whatever?
+#export RELEASE_BUILD=${RELEASE_BUILD:-true}
+RELEASE_BUILD=false
+#
+export TEMPEST_TAG="${TEMPEST_TAG:-mcp/pike}"
+export HORIZON_TAG="${HORIZON_TAG:-mcp/pike}"
+export BARBICAN_TAG="${BARBICAN_TAG:-mcp/pike}"
+export DESIGNATE_TAG="${DESIGNATE_TAG:-mcp/pike}"
+export HEAT_TAG="${HEAT_TAG:-mcp/pike}"
+export IRONIC_TAG="${IRONIC_TAG:-mcp/pike}"
+export MANILA_TAG="${MANILA_TAG:-mcp/pike}"
+export TELEMETRY_TAG="${TELEMETRY_TAG:-mcp/pike}"
+export OCTAVIA_TAG="${OCTAVIA_TAG:-mcp/pike}"
+export PATROLE_TAG="${PATROLE_TAG:-0.4.x}"
+export TUNGSTEN_TAG="${TUNGSTEN_TAG:-0.1.x}"
+# global openstack requirements repo
+# This is the commit where tempest version is 17.2.0 in u-c
+export UPPER_CONSTRAINTS_TAG="${UPPER_CONSTRAINTS_TAG:-83b1de74834fbcdbe749e270c29e76cdc28f85d5}"
+
+function prepare(){
+ # TODO migrate to downstrem mirror?
+ mkdir -p data; pushd data
+ if [ ! -d openstack_requirements ] ; then
+ git clone https://github.com/openstack/requirements.git openstack_requirements
+ pushd openstack_requirements; git checkout $UPPER_CONSTRAINTS_TAG; popd
+ fi
+ sed -i -e "s|tempest=.*|git+file:///var/lib/tempest@${TEMPEST_TAG}#egg=tempest|g" \
+ openstack_requirements/upper-constraints.txt;
+ #
+ if [ ! -d tempest-horizon-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/tempest-horizon tempest-horizon-plugin
+ pushd tempest-horizon-plugin ; git checkout $HORIZON_TAG; popd
+ fi
+ #
+ if [ ! -d barbican-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/barbican-tempest-plugin
+ pushd barbican-tempest-plugin; git checkout $BARBICAN_TAG; popd
+ fi
+ #
+ if [ ! -d designate-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/designate-tempest-plugin
+ pushd designate-tempest-plugin; git checkout $DESIGNATE_TAG; popd;
+ fi
+ #
+ if [ ! -d heat-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/heat-tempest-plugin
+ pushd heat-tempest-plugin; git checkout $HEAT_TAG; popd
+ fi
+ #
+ if [ ! -d ironic-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/ironic-tempest-plugin
+ pushd ironic-tempest-plugin; git checkout $IRONIC_TAG; popd;
+ fi
+ #
+ if [ ! -d manila-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/manila-tempest-plugin
+ pushd manila-tempest-plugin; git checkout $MANILA_TAG; popd;
+ fi
+ #
+ if [ ! -d telemetry-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/telemetry-tempest-plugin
+ pushd telemetry-tempest-plugin; git checkout $TELEMETRY_TAG; popd;
+ fi
+ #
+ if [ ! -d octavia-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/octavia-tempest-plugin
+ pushd octavia-tempest-plugin; git checkout $OCTAVIA_TAG; popd;
+ fi
+ #
+ if [ ! -d patrole-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/patrole patrole-tempest-plugin
+ pushd patrole-tempest-plugin; git checkout $PATROLE_TAG; popd;
+ fi
+ #
+ if [ ! -d tungsten-tempest-plugin ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/contrail/tungsten-tempest tungsten-tempest-plugin
+ pushd tungsten-tempest-plugin; git checkout $TUNGSTEN_TAG; popd;
+ fi
+ #
+ if [ ! -d tempest ] ; then
+ git clone ssh://${GERRIT_NAME}@gerrit.mcp.mirantis.com:29418/packaging/sources/tempest tempest
+ pushd tempest ; git checkout $TEMPEST_TAG; popd
+ fi
+ #
+ if [[ "$(echo ${RELEASE_BUILD} | tr [:upper:] [:lower:] )" == "true" ]]; then
+ find . -maxdepth 2 -type d -iname .git -not -path '*tempest/.git*' | xargs --no-run-if-empty rm -rf
+ find . -maxdepth 2 -type f -iname .gitignore -or -iname .gitreview | xargs --no-run-if-empty rm -v
+ fi
+ popd
+ }
+
+function buildme(){
+ echo "=== Installing plugins"
+ plugins="tempest-horizon-plugin barbican-tempest-plugin designate-tempest-plugin heat-tempest-plugin ironic-tempest-plugin"
+ plugins="${plugins} manila-tempest-plugin telemetry-tempest-plugin octavia-tempest-plugin patrole-tempest-plugin tungsten-tempest-plugin"
+ for plugin in ${plugins}; do
+ echo "=== Installing plugin: ${plugin}"
+ pushd /var/lib/${plugin}
+ pip install --constraint /var/lib/openstack_requirements/upper-constraints.txt -r requirements.txt .
+ popd
+ done
+ echo "=== Installing tempest itself"
+ pushd /var/lib/tempest/
+ pip install .
+ popd
+ pip install junitxml
+ echo "BUILD_TIMESTAMP=$(date '+%Y-%m-%d-%H-%M-%S' -u)" > /etc/build_timestamp
+}
+
+############# body ############################################################
+
+case "$1" in
+
+buildme) echo "Attempt to buildme"
+ buildme
+ ;;
+*) echo "Attempt to prepare.."
+ prepare
+ ;;
+esac
diff --git a/pre-build.sh b/pre-build.sh
new file mode 120000
index 0000000..005ff1e
--- /dev/null
+++ b/pre-build.sh
@@ -0,0 +1 @@
+extra/pre-build.sh
\ No newline at end of file