Merge "packer-image-buid updates"
diff --git a/common/ubuntu_base.sh b/common/ubuntu_base.sh
index 6f314e9..336662b 100644
--- a/common/ubuntu_base.sh
+++ b/common/ubuntu_base.sh
@@ -1,7 +1,11 @@
 #!/bin/bash -xe
 
+if [ -f /tmp/done_ubuntu_base ] ; then
+  echo "INFO: ubuntu_base already finished.Skipping.."
+  exit 0
+fi
+#
 UBUNTU_BASEURL="${UBUNTU_BASEURL:-mirror://mirrors.ubuntu.com/mirrors.txt}"
-
 ## Base packages and setup
 export DEBIAN_FRONTEND=noninteractive
 echo -e '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d
@@ -32,13 +36,16 @@
 apt-get update
 
 # Useful tools
-apt-get -y install byobu curl ethtool htop iputils-ping lsof strace tcpdump tmux traceroute tree vim-nox wget
-# Install common prerequisites
-apt-get -y install apt-transport-https libmnl0 python-apt python-m2crypto python-psutil acpid
+EXTRA_PKGS="byobu curl ethtool iputils-ping lsof strace tcpdump traceroute wget iptables"
+# Pretty tools
+EXTRA_PKGS="${EXTRA_PKGS} byobu htop tmux tree vim-nox mc"
+# Common prerequisites
+EXTRA_PKGS="${EXTRA_PKGS} apt-transport-https libmnl0 python-apt python-m2crypto python-psutil acpid"
+apt-get -y install ${EXTRA_PKGS}
 
 # Cleanup old kernels, ensure latest is installed via virtual package
 if [ ! -f /tmp/no_install_kernel ]; then
-    apt-get purge -y linux-image-* linux-headers-*
+    apt-get purge -y linux-image-* linux-headers-* | grep -v 'is not installed, so not removed'
     apt-get install -y linux-image-virtual-lts-xenial linux-image-extra-virtual-lts-xenial
 
     # Update grub cmdline
@@ -69,3 +76,5 @@
   systemctl disable ${s} || true
   systemctl stop ${s} || true
 done
+
+touch /tmp/done_ubuntu_base
diff --git a/common/ubuntu_salt_bootstrap.sh b/common/ubuntu_salt_bootstrap.sh
index ad3757e..0b45762 100644
--- a/common/ubuntu_salt_bootstrap.sh
+++ b/common/ubuntu_salt_bootstrap.sh
@@ -1,5 +1,10 @@
 #!/bin/bash -xe
 
+if [ -f '/tmp/done_ubuntu_salt_bootstrap' ]; then
+  echo "INFO: ubuntu_salt_bootstrap already finished! Skipping.."
+  exit 0
+fi
+#
 CLUSTER_NAME=${CLUSTER_NAME:-lost_cluster_name_variable}
 CLUSTER_MODEL=${CLUSTER_MODEL:-https://github.com/Mirantis/mcp-offline-model.git}
 CLUSTER_MODEL_REF=${CLUSTER_MODEL_REF:-master}
@@ -43,6 +48,7 @@
 export EXTRA_FORMULAS=${EXTRA_FORMULAS:-"ntp aptly nginx iptables docker git maas jenkins"}
 export APT_REPOSITORY=" deb [arch=amd64] ${APT_MIRANTIS_SALT_REPO} "
 export APT_REPOSITORY_GPG=${APT_MIRANTIS_GPG}
+export SALT_STOPSTART_WAIT=${SALT_STOPSTART_WAIT:-10}
 if [[ ! -z "${PACKER_OFFLINE_BUILD}" ]];then
   echo "INFO: build in offline detected!"
   export BOOTSTRAP_SALTSTACK_COM="file:///tmp/bootstrap.saltstack.com.sh"
@@ -56,3 +62,4 @@
   exit 1
 fi
 bash -x /srv/salt/scripts/bootstrap.sh || true
+touch /tmp/done_ubuntu_salt_bootstrap
diff --git a/day01-image/run.example.sh b/day01-image/run.example.sh
index aeee71e..25efd13 100755
--- a/day01-image/run.example.sh
+++ b/day01-image/run.example.sh
@@ -21,14 +21,17 @@
 export FORMULA_VERSION=proposed
 export BINARY_MCP_VERSION=proposed
 export UBUNTU_BASEURL="http://mirror.mirantis.com/proposed/ubuntu/"
-export SALTSTACK_REPO="http://mirror.mirantis.com/proposed/saltstack-2016.3/xenial xenial main"
+export SALTSTACK_REPO="http://mirror.mirantis.com/proposed/saltstack-2017.7/xenial xenial main"
 export APT_MIRANTIS_GPG="http://apt.mirantis.com/public.gpg"
-export SALTSTACK_GPG="https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3/SALTSTACK-GPG-KEY.pub"
+export SALTSTACK_GPG="https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2017.7/SALTSTACK-GPG-KEY.pub"
 export APT_MIRANTIS_SALT_REPO="http://apt.mirantis.com/xenial/ proposed salt"
 export GIT_SALT_FORMULAS_SCRIPTS=https://github.com/salt-formulas/salt-formulas-scripts.git
 export APT_REPOSITORY="deb [arch=amd64] http://apt.mirantis.com/xenial/ proposed salt"
 export APT_REPOSITORY_GPG=http://apt.mirantis.com/public.gpg
 ###
+# Hard-coded folder in template
+export PACKER_IMAGES_CACHE="${HOME}/packer_images_cache/"
+mkdir -p "${PACKER_IMAGES_CACHE}"
 
 export PACKER_LOG=1
 # For qemu test-build:
diff --git a/day01-image/scripts/cleanup.sh b/day01-image/scripts/cleanup.sh
index 2b5ccbc..0f5adec 100644
--- a/day01-image/scripts/cleanup.sh
+++ b/day01-image/scripts/cleanup.sh
@@ -10,6 +10,10 @@
 echo > /etc/apt/sources.list  || true
 rm -vf /usr/sbin/policy-rc.d || true
 
+
+echo "removeing all previously accepted salt keys"
+salt-key -D -y || true
+
 echo "cleaning up reclass"
 rm -rf /srv/salt/reclass || true
 rm -rf /srv/salt/scripts || true
@@ -42,7 +46,7 @@
 
 # stop and disable services, for healthy zerodisk
 # They should be enabled after cfg01 init
-stop_services="postgresql.service salt-api salt-master salt-minion jenkins maas-rackd.service maas-regiond.service"
+stop_services="postgresql.service salt-api salt-master salt-minion jenkins maas-rackd.service maas-regiond.service bind9"
 for s in ${stop_services} ; do
   systemctl stop ${s} || true
   systemctl disable ${s} || true
diff --git a/day01-image/scripts/salt.sh b/day01-image/scripts/salt.sh
index 78cdd79..a49b274 100644
--- a/day01-image/scripts/salt.sh
+++ b/day01-image/scripts/salt.sh
@@ -17,6 +17,7 @@
 salt-call ${SALT_OPTS} state.sls linux.network
 salt-call ${SALT_OPTS} state.sls openssh
 salt-call ${SALT_OPTS} state.sls git.server
+sleep 99h || true
 salt-call ${SALT_OPTS} state.sls postgresql
 # install only MAAS packages, don't configure anything.
 # Configuration will be processed during cfg01 initialization.
diff --git a/day01-image/template.json b/day01-image/template.json
index 1406680..c4ff8de 100644
--- a/day01-image/template.json
+++ b/day01-image/template.json
@@ -2,6 +2,8 @@
   "variables": {
     "user": "root",
     "password": "r00tme",
+    "images_cache": "{{ env `PACKER_IMAGES_CACHE` }}",
+    "disk_size": "50000",
     "do_api_token": "{{ env `DO_API_TOKEN` }}",
     "image_name": "{{ env `IMAGE_NAME` }}",
     "cluster_model": "{{ env `CLUSTER_MODEL` }}",
@@ -17,8 +19,7 @@
     "apt_mirantis_salt_repo": "{{ env `APT_MIRANTIS_SALT_REPO` }}",
     "git_salt_formulas_scripts": "{{ env `GIT_SALT_FORMULAS_SCRIPTS` }}",
     "apt_repository": "{{ env `APT_REPOSITORY` }}",
-    "apt_repository_gpg": "{{ env `APT_REPOSITORY_GPG` }}",
-    "disk_size": "50000"
+    "apt_repository_gpg": "{{ env `APT_REPOSITORY_GPG` }}"
 
   },
   "provisioners": [
@@ -125,6 +126,7 @@
       "iso_checksum": "566efef1d6f12e7d3a994c2405bdb642",
       "iso_checksum_type": "md5",
       "iso_url": "http://cloud-images.ubuntu.com/releases/xenial/release-20180306/ubuntu-16.04-server-cloudimg-amd64-disk1.img",
+      "iso_target_path": "{{ user `images_cache`}}/release-20180306_xenial-server-cloudimg-amd64-disk1.img",
       "disk_image": true,
       "accelerator": "kvm",
       "disk_size": "{{ user `disk_size`}}",
diff --git a/day01-image/template.json.env b/day01-image/template.json.env
new file mode 100644
index 0000000..fb05cc3
--- /dev/null
+++ b/day01-image/template.json.env
@@ -0,0 +1 @@
+PACKER_IMAGES_CACHE="${HOME}/packer_images_cache/"
diff --git a/mirror-image/run.example.sh b/mirror-image/run.example.sh
index 3f7328c..cab6393 100644
--- a/mirror-image/run.example.sh
+++ b/mirror-image/run.example.sh
@@ -12,6 +12,8 @@
 # http/bootstrap.saltstack.com.sh https://github.com/saltstack/salt-bootstrap
 #
 
+export PACKER_IMAGES_CACHE="${HOME}/packer_images_cache/"
+#
 export CLUSTER_MODEL="https://github.com/Mirantis/mcp-offline-model.git"
 export CLUSTER_MODEL_REF="master"
 export CLUSTER_NAME="mcp-offline"
@@ -21,9 +23,9 @@
 
 BINARY_MCP_VERSION="nightly"
 export UBUNTU_BASEURL="http://mirror.mirantis.com/${BINARY_MCP_VERSION}/ubuntu/"
-export SALTSTACK_REPO="http://apt.mirantis.com/xenial/salt/2016.3/ ${BINARY_MCP_VERSION} main"
+export SALTSTACK_REPO="http://mirror.mirantis.com/${BINARY_MCP_VERSION}/saltstack-2017.7/xenial xenial main"
+export SALTSTACK_GPG="http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2017.7/SALTSTACK-GPG-KEY.pub"
 export APT_MIRANTIS_GPG="http://apt.mirantis.com/public.gpg"
-export SALTSTACK_GPG="${APT_MIRANTIS_GPG}"
 export APT_MIRANTIS_SALT_REPO="http://apt.mirantis.com/xenial/ ${BINARY_MCP_VERSION} salt "
 #
 export GIT_SALT_FORMULAS_SCRIPTS="https://github.com/salt-formulas/salt-formulas-scripts.git"
@@ -32,20 +34,20 @@
 export APT_REPOSITORY_GPG=${APT_MIRANTIS_GPG}
 
 # Openstack-related variables
-export VM_FLAVOR="3c4fe514-3eb6-464d-a419-7be09d72cf1e"
-export VM_FLOATING_IP_POOL="095f43ed-65ba-48ce-8185-d4080e430a88"
-export IMAGE_NAME="mcp-offline-mirror-$(date '+%Y-%m-%d-%H-%M-%S')"
-export VM_IMAGE="9953a40d-1fa2-4f26-b146-0507dbc912dc" # ubuntu cloud img
+#export VM_FLAVOR="3c4fe514-3eb6-464d-a419-7be09d72cf1e"
+#export VM_FLOATING_IP_POOL="095f43ed-65ba-48ce-8185-d4080e430a88"
+#export IMAGE_NAME="mcp-offline-mirror-$(date '+%Y-%m-%d-%H-%M-%S')"
+#export VM_IMAGE="9953a40d-1fa2-4f26-b146-0507dbc912dc" # ubuntu cloud img
 # May be need, if cloud with keystone v3 used
 #export OS_TENANT_ID=$OS_PROJECT_ID
 #export OS_DOMAIN_NAME="xxxx"
 #export OS_TENANT_NAME="xxxxx"
 
+# Hard-coded folder in template
+mkdir -p "${PACKER_IMAGES_CACHE}"
+
 export PACKER_LOG=1
 # For qemu test-build:
-#cloud-localds  --hostname ubuntu --dsmode local config-drive/cloudata.iso  config-drive/user-data.yaml
-# packer build -only=qemu -parallel=false -on-error=ask template.json
+cloud-localds  --hostname ubuntu --dsmode local config-drive/cloudata.iso  config-drive/user-data.yaml
+packer build -only=qemu -parallel=false -on-error=ask template.json
 # rm -rf ~/.packer.d/
-
-# For openstack test-build
-packer build -only=openstack -parallel=false -on-error=ask template.json
diff --git a/mirror-image/scripts/mirrors.sh b/mirror-image/scripts/mirrors.sh
index fe7101f..764597a 100644
--- a/mirror-image/scripts/mirrors.sh
+++ b/mirror-image/scripts/mirrors.sh
@@ -1,19 +1,22 @@
 #!/bin/bash -xe
 
+SALT_OPTS="-l debug -t 10 --retcode-passthrough --no-color"
+
 #aptly
-salt-call -t 5 --no-color state.sls aptly.server
-salt-call -t 5 --no-color state.sls aptly.publisher
-salt-call -t 5 --no-color state.sls aptly
-salt-call -t 5 --retcode-passthrough --no-color state.sls aptly
+# FIXME: remove duplicate run after fix: PROD-21387
+# RC: aptly may fail to import gpg key's randomly(most probably,nw issue), so lets run it twice
+salt-call ${SALT_OPTS} state.sls aptly.server || salt-call ${SALT_OPTS} state.sls aptly.server
+salt-call ${SALT_OPTS} state.sls aptly.publisher
+salt-call ${SALT_OPTS} state.sls aptly || salt-call ${SALT_OPTS} state.sls aptly
 aptly_mirror_update.sh -sv
 aptly-publisher --url http://10.99.0.1:18084 --timeout=1200 publish -v -d -c /etc/aptly-publisher.yaml --architectures amd64 --recreate --force-overwrite
 
 #docker registry
-salt-call -t 5 --retcode-passthrough state.sls docker.client.registry
-docker system prune --all --force
+salt-call ${SALT_OPTS} state.sls docker.client.registry
+docker system prune --all --force || true
 
 #debmirror
-salt-call -t 5 --no-color state.sls debmirror
+salt-call ${SALT_OPTS} state.sls debmirror
 
 #git
-salt-call -t 5 --retcode-passthrough --no-color state.sls git.server
+salt-call ${SALT_OPTS} state.sls git.server
diff --git a/mirror-image/scripts/salt.sh b/mirror-image/scripts/salt.sh
index 613352a..9655ec0 100644
--- a/mirror-image/scripts/salt.sh
+++ b/mirror-image/scripts/salt.sh
@@ -1,16 +1,19 @@
 #!/bin/bash -xe
 
+SALT_OPTS="-l debug -t 10 --retcode-passthrough --no-color"
+
 echo "deb [arch=amd64] ${APT_MIRANTIS_SALT_REPO}" > /etc/apt/sources.list.d/mcp_salt.list
 apt-get update
 apt-get install salt-formula* -y
 salt-call saltutil.refresh_pillar
 salt-call saltutil.sync_all
-salt-call -t 5 --retcode-passthrough --no-color state.sls salt
-salt-call -t 5 --retcode-passthrough --no-color state.sls linux.system.repo,linux.system.directory,linux.system.package
-salt-call -t 5 --retcode-passthrough --no-color state.sls linux.system.file
-salt-call -t 5 --retcode-passthrough --no-color state.sls docker.host
-salt-call -t 5 --retcode-passthrough --no-color state.sls docker exclude=docker.client.registry
+
+salt-call ${SALT_OPTS} state.sls salt
+salt-call ${SALT_OPTS} state.sls linux.system.repo,linux.system.directory,linux.system.package
+salt-call ${SALT_OPTS} state.sls linux.system.file
+salt-call ${SALT_OPTS} state.sls docker.host
+salt-call ${SALT_OPTS} state.sls docker exclude=docker.client.registry
 sleep 120
-salt-call -t 5 --retcode-passthrough --no-color state.sls linux.network
-salt-call -t 5 --retcode-passthrough --no-color state.sls nginx
-salt-call -t 5 --retcode-passthrough --no-color state.sls maas.mirror
+salt-call ${SALT_OPTS} state.sls linux.network
+salt-call ${SALT_OPTS} state.sls nginx
+salt-call ${SALT_OPTS} state.sls maas.mirror
diff --git a/mirror-image/template.json b/mirror-image/template.json
index 5abc542..5b942d0 100644
--- a/mirror-image/template.json
+++ b/mirror-image/template.json
@@ -3,6 +3,7 @@
     "user": "root",
     "password": "r00tme",
     "disk_size": "100000",
+    "images_cache": "{{ env `PACKER_IMAGES_CACHE` }}",
     "do_api_token": "{{ env `DO_API_TOKEN` }}",
     "image_name": "{{ env `IMAGE_NAME` }}",
     "cluster_model": "{{ env `CLUSTER_MODEL` }}",
@@ -151,7 +152,7 @@
       "accelerator": "kvm",
       "disk_size": "{{ user `disk_size`}}",
       "iso_url": "http://cloud-images.ubuntu.com/releases/xenial/release-20180306/ubuntu-16.04-server-cloudimg-amd64-disk1.img",
-      "iso_target_path": "/root/isos/xenial-server-cloudimg-amd64-disk1.img",
+      "iso_target_path": "{{ user `images_cache`}}/release-20180306_xenial-server-cloudimg-amd64-disk1.img",
       "iso_checksum_type": "md5",
       "disk_image": true,
       "disk_compression": true,
@@ -160,7 +161,7 @@
       "headless": true,
       "ssh_username": "{{user `user`}}",
       "ssh_password": "{{user `password`}}",
-      "shutdown_command": "sync; shutdown -P now",
+      "shutdown_command": "sync ; shutdown -P now",
       "vnc_bind_address": "0.0.0.0",
       "vnc_port_max": "5956",
       "vnc_port_min": "5956",
diff --git a/ubuntu-16.04/run.example.sh b/ubuntu-16.04/run.example.sh
new file mode 100644
index 0000000..df58f50
--- /dev/null
+++ b/ubuntu-16.04/run.example.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# pre-requirments:
+# apt-get install cloud-localds
+# Cloudimg: wget https://cloud-images.ubuntu.com/xenial/XXXX/xenial-server-cloudimg-amd64-disk1.img
+# Packer: https://releases.hashicorp.com/packer/1.1.3/packer_1.1.3_linux_amd64.zip
+
+# Those script - only example for variables, which should be passed to packer and
+# overwrite variables under /scripts/ directory
+
+# External script sources:
+# http/bootstrap.saltstack.com.sh https://github.com/saltstack/salt-bootstrap
+#
+export IMAGE_NAME="ubuntu-16-04-x64"
+export UBUNTU_BASEURL="http://mirror.mirantis.com/proposed/ubuntu/"
+export SALTSTACK_REPO="deb [arch=amd64] http://mirror.mirantis.com/proposed/saltstack-2017.7/xenial xenial main"
+export SALTSTACK_GPG="https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2017.7/SALTSTACK-GPG-KEY.pub"
+###
+# Hard-coded folder in template
+export PACKER_IMAGES_CACHE="${HOME}/packer_images_cache/"
+mkdir -p "${PACKER_IMAGES_CACHE}"
+
+export PACKER_LOG=1
+# For qemu test-build:
+cloud-localds  --hostname ubuntu --dsmode local config-drive/cloudata.iso  config-drive/user-data.yaml
+packer build -only=qemu -parallel=false -on-error=ask template.json
+#rm -rf ~/.packer.d/
+
diff --git a/ubuntu-16.04/template.json b/ubuntu-16.04/template.json
index 84e1a59..284237e 100644
--- a/ubuntu-16.04/template.json
+++ b/ubuntu-16.04/template.json
@@ -4,6 +4,7 @@
     "password": "ho5uo7Uome5d",
     "distro": "ubuntu-16-04-x64",
     "disk_size": "8000",
+    "images_cache": "{{ env `PACKER_IMAGES_CACHE` }}",
     "ubuntu_baseurl": "https://mirror.mirantis.com/proposed/ubuntu/",
     "saltstack_gpg": "{{ env `SALTSTACK_GPG` }}",
     "saltstack_repo": "{{ env `SALTSTACK_REPO` }}",
@@ -284,6 +285,7 @@
       "iso_checksum": "566efef1d6f12e7d3a994c2405bdb642",
       "iso_checksum_type": "md5",
       "iso_url": "http://cloud-images.ubuntu.com/releases/xenial/release-20180306/ubuntu-16.04-server-cloudimg-amd64-disk1.img",
+      "iso_target_path": "{{ user `images_cache`}}/release-20180306_xenial-server-cloudimg-amd64-disk1.img",
       "disk_image": true,
       "disk_compression": true,
       "accelerator": "kvm",
diff --git a/ubuntu-16.04/template.json.env b/ubuntu-16.04/template.json.env
index 12be33e..ed6351e 100644
--- a/ubuntu-16.04/template.json.env
+++ b/ubuntu-16.04/template.json.env
@@ -1,5 +1,7 @@
 # Env variables and description:
-DO_API_TOKEN
+#DO_API_TOKEN
+PACKER_IMAGES_CACHE="${HOME}/packer_images_cache/"
+
 
 # base.sh
 UBUNTU_BASEURL=http://mirror.mirantis.com/stable/ubuntu/