Refactor,update images
* Switch ubuntu|salt-stack to fetch from mirror.mirantis.com/proposed
* ubuntu-16-04 - add openstack builder[inactive stub]
* add info.sh - and enable it in mirror and ubuntu16
* Drop mcp salt repos - we don't use them for build
* switch to release-20180306 ubuntu cloud image
Change-Id: Ic45db3fd2579262b055ecdc54b27a16a837c2dd4
diff --git a/common/ubuntu_base.sh b/common/ubuntu_base.sh
index bae5120..aafae3a 100644
--- a/common/ubuntu_base.sh
+++ b/common/ubuntu_base.sh
@@ -4,9 +4,13 @@
## Base packages and setup
export DEBIAN_FRONTEND=noninteractive
-echo "exit 101" > /usr/sbin/policy-rc.d
+echo -e '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
+# Configure apt. Please refer to
+# https://github.com/Mirantis/reclass-system-salt-model/blob/master/linux/system/single/debian.yml
+# and keep those structures with same naming convention - to prevent
+# misconfiguration between base system and salt state.
echo "Acquire::CompressionTypes::Order gz;" >/etc/apt/apt.conf.d/99compression-workaround-salt
echo "Acquire::EnableSrvRecords false;" >/etc/apt/apt.conf.d/99enablesrvrecords-false
echo "Acquire::http::Pipeline-Depth 0;" > /etc/apt/apt.conf.d/99aws-s3-mirrors-workaround-salt
@@ -30,7 +34,7 @@
# Useful tools
apt-get -y install byobu curl ethtool htop iputils-ping lsof strace tcpdump tmux traceroute tree vim-nox wget
# Install common prerequisites
-apt-get -y install apt-transport-https libmnl0 python-apt python-m2crypto python-psutil
+apt-get -y install apt-transport-https libmnl0 python-apt python-m2crypto python-psutil acpid
# Cleanup old kernels, ensure latest is installed via virtual package
if [ ! -f /tmp/no_install_kernel ]; then
diff --git a/common/ubuntu_cleanup.sh b/common/ubuntu_cleanup.sh
index 857c7d9..f043e26 100644
--- a/common/ubuntu_cleanup.sh
+++ b/common/ubuntu_cleanup.sh
@@ -1,4 +1,5 @@
#!/bin/bash -xe
+
apt-get -y autoremove --purge
apt-get -y clean
rm -rf /var/lib/apt/lists/*
@@ -24,4 +25,8 @@
rm -f /etc/salt/minion_id || true
echo "cleaning up resolvconf"
-sed -i '/172\.18\.208\.44/d' /etc/resolvconf/resolv.conf.d/base
\ No newline at end of file
+sed -i '/172\.18\.208\.44/d' /etc/resolvconf/resolv.conf.d/base
+
+# Clear\drop cache's
+sync
+echo 3 > /proc/sys/vm/drop_caches
diff --git a/common/ubuntu_info.sh b/common/ubuntu_info.sh
new file mode 100644
index 0000000..903c121
--- /dev/null
+++ b/common/ubuntu_info.sh
@@ -0,0 +1,5 @@
+#!/bin/bash -xe
+
+# Save some basic debug information.
+
+dpkg-query -W -f='${Package}=${Version}\n' >> /var/log/vcp_initial_pkgs.log
diff --git a/mirror-image/template.json b/mirror-image/template.json
index fd65d1e..24875e3 100644
--- a/mirror-image/template.json
+++ b/mirror-image/template.json
@@ -2,9 +2,9 @@
"variables": {
"user": "root",
"password": "r00tme",
+ "disk_size": "150000",
"do_api_token": "{{ env `DO_API_TOKEN` }}",
"image_name": "{{ env `IMAGE_NAME` }}",
- "disk_size": "150000",
"cluster_model": "{{ env `CLUSTER_MODEL` }}",
"cluster_model_ref": "{{ env `CLUSTER_MODEL_REF` }}",
"cluster_name": "{{ env `CLUSTER_NAME` }}",
@@ -18,11 +18,11 @@
"git_salt_formulas_scripts": "{{ env `GIT_SALT_FORMULAS_SCRIPTS` }}",
"apt_repository": "{{ env `APT_REPOSITORY` }}",
"apt_repository_gpg": "{{ env `APT_REPOSITORY_GPG` }}",
- "os_endpoint": "https://cloud-cz.bud.mirantis.net:5000/v2.0",
"os_flavor": "56c551be-41c0-49a4-b367-9b31691c32c3",
"os_username": "{{ env `OS_USERNAME` }}",
"os_password": "{{ env `OS_PASSWORD` }}",
- "os_image": "60878bd4-cb4a-4d71-ae02-2a8ee4476d10",
+ "os_image": "7451fac0-0ce5-4d30-ae36-cd9a90e7948e",
+ "os_endpoint": "https://cloud-cz.bud.mirantis.net:5000/v2.0",
"os_zone": "mcp-mk",
"os_floating_ip_pool": "public",
"os_tenant": "26e4dd19485249608ee3685f254f3909",
@@ -93,6 +93,7 @@
"scripts": [
"scripts/mirrors.sh",
"scripts/security.sh",
+ "scripts/info.sh",
"scripts/cleanup.sh"
]
},
diff --git a/ubuntu-16.04/config-drive/user-data.yaml b/ubuntu-16.04/config-drive/user-data.yaml
index 32a896e..110d9ad 100644
--- a/ubuntu-16.04/config-drive/user-data.yaml
+++ b/ubuntu-16.04/config-drive/user-data.yaml
@@ -11,3 +11,8 @@
runcmd:
- sed -i'.orig' -e's/PermitRootLogin.*/PermitRootLogin yes/g' -e's/PasswordAuthentication.*/PasswordAuthentication yes/g' /etc/ssh/sshd_config
- service sshd restart
+# speed up resolving, and not stuck on timeouts. One-shot changes
+ - echo '127.0.0.1 ubuntu' >> /etc/hosts
+ - echo 'options timeout:1 attempts:4' >> /etc/resolv.conf
+ - echo 'nameserver 172.18.208.44' >> /etc/resolvconf/resolv.conf.d/base
+ - resolvconf -u
diff --git a/ubuntu-16.04/scripts/info.sh b/ubuntu-16.04/scripts/info.sh
new file mode 120000
index 0000000..772d054
--- /dev/null
+++ b/ubuntu-16.04/scripts/info.sh
@@ -0,0 +1 @@
+../../common/ubuntu_info.sh
\ No newline at end of file
diff --git a/ubuntu-16.04/scripts/salt.sh b/ubuntu-16.04/scripts/salt.sh
index dd13dfd..2ddedc4 100644
--- a/ubuntu-16.04/scripts/salt.sh
+++ b/ubuntu-16.04/scripts/salt.sh
@@ -1,16 +1,19 @@
#!/bin/bash -xe
-# Add apt repository
-echo "deb [arch=amd64] http://apt.mirantis.com/xenial/ nightly salt" > /etc/apt/sources.list.d/mcp_salt.list
-echo "deb [arch=amd64] http://apt.mirantis.com/xenial/ nightly extra" > /etc/apt/sources.list.d/mcp_extra.list
-curl -sf http://apt.mirantis.com/public.gpg | apt-key add -
+SALTSTACK_REPO=${SALTSTACK_REPO:-"deb [arch=amd64] https://mirror.mirantis.com/stable/saltstack-2016.3/xenial/ xenial main"}
+SALTSTACK_GPG=${SALTSTACK_GPG:-"https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3/SALTSTACK-GPG-KEY.pub"}
-wget -O - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3/SALTSTACK-GPG-KEY.pub | sudo apt-key add -
-echo "deb http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3 xenial main" >/etc/apt/sources.list.d/saltstack.list
+wget -O - ${SALTSTACK_GPG} | sudo apt-key add -
+echo "${SALTSTACK_REPO}" > /etc/apt/sources.list.d/saltstack.list
+
+# We expect issues with GPG,related to our mirrors - so temporary disable check.
+echo "APT::Get::AllowUnauthenticated true;" > /etc/apt/apt.conf.d/99allow_unauthenticated
+
apt-get update
apt-get -y install salt-minion
mkdir -m700 -p /etc/salt/pki/minion
-echo "
+
+cat <<EOF >> /etc/salt/minion.d/minion.conf
max_event_size: 100000000
acceptance_wait_time_max: 60
acceptance_wait_time: 10
@@ -18,4 +21,8 @@
recon_default: 1000
recon_max: 60000
recon_randomize: True
-auth_timeout: 60" >> /etc/salt/minion
+auth_timeout: 60
+EOF
+
+rm -vf /etc/apt/apt.conf.d/99allow_unauthenticated
+rm -vf /etc/apt/sources.list.d/saltstack.list
diff --git a/ubuntu-16.04/scripts/zerodisk.sh b/ubuntu-16.04/scripts/zerodisk.sh
index 6a658ed..f2c6f16 100644
--- a/ubuntu-16.04/scripts/zerodisk.sh
+++ b/ubuntu-16.04/scripts/zerodisk.sh
@@ -1,2 +1,3 @@
dd if=/dev/zero of=/EMPTY bs=1M || true
rm -f /EMPTY
+sync
diff --git a/ubuntu-16.04/template.json b/ubuntu-16.04/template.json
index c779c11..400a237 100644
--- a/ubuntu-16.04/template.json
+++ b/ubuntu-16.04/template.json
@@ -2,12 +2,27 @@
"variables": {
"user": "root",
"password": "ho5uo7Uome5d",
- "do_api_token": "{{ env `DO_API_TOKEN` }}",
"distro": "ubuntu-16-04-x64",
- "disk_size": "8000"
+ "disk_size": "8000",
+ "ubuntu_baseurl": "https://mirror.mirantis.com/proposed/",
+ "saltstack_repo": "deb [arch=amd64] https://mirror.mirantis.com/proposed/saltstack-2016.3/xenial/ xenial main",
+ "do_api_token": "{{ env `DO_API_TOKEN` }}",
+ "image_name": "ubuntu-16-04-x64",
+ "os_username": "{{ env `OS_USERNAME` }}",
+ "os_password": "{{ env `OS_PASSWORD` }}",
+ "os_image": "7451fac0-0ce5-4d30-ae36-cd9a90e7948e",
+ "os_endpoint": "https://cloud-cz.bud.mirantis.net:5000/v2.0",
+ "os_zone": "mcp-mk",
+ "os_floating_ip_pool": "public",
+ "os_tenant": "26e4dd19485249608ee3685f254f3909",
+ "os_network": "772f8ca6-1f4a-4535-99d0-880d6ead9a82"
},
"provisioners": [
{
+ "environment_vars": [
+ "UBUNTU_BASEURL={{ user `ubuntu_baseurl` }}",
+ "SALTSTACK_GPG={{ user `saltstack_gpg` }}"
+ ],
"type": "shell",
"execute_command": "sh '{{.Path}}'",
"override": {
@@ -43,9 +58,21 @@
"scripts/network.sh",
"scripts/security.sh",
"scripts/cleanup.sh",
+ "scripts/info.sh",
"scripts/zerodisk.sh"
]
},
+ "openstack": {
+ "scripts": [
+ "scripts/base.sh",
+ "scripts/motd.sh",
+ "scripts/salt.sh",
+ "scripts/network.sh",
+ "scripts/security.sh",
+ "scripts/cleanup.sh",
+ "scripts/info.sh"
+ ]
+ },
"docker": {
"scripts": [
"scripts/docker.sh",
@@ -229,13 +256,26 @@
}
},
{
+ "type": "docker",
+ "image": "ubuntu:16.04",
+ "commit": true
+ },
+ {
+ "type": "digitalocean",
+ "api_token": "{{user `do_api_token`}}",
+ "image": "{{user `distro`}}",
+ "snapshot_name": "{{ user `distro` }}-{{ isotime \"200601021504\" }}",
+ "region": "ams2",
+ "size": "1gb"
+ },
+ {
"type": "qemu",
"qemuargs": [
[ "-m", "1024M" ],
[ "-fda", "config-drive/cloudata.iso" ]
],
- "vm_name": "{{ user `distro` }}-{{ isotime \"200601021504\" }}",
- "output_directory": "images/{{ user `distro` }}-qemu-{{ isotime \"200601021504\" }}",
+ "vm_name": "{{ user `image_name` }}-{{ isotime \"200601021504\" }}",
+ "output_directory": "images/{{ user `image_name` }}-qemu-{{ isotime \"200601021504\" }}",
"format": "qcow2",
"iso_checksum": "566efef1d6f12e7d3a994c2405bdb642",
"iso_checksum_type": "md5",
@@ -251,17 +291,21 @@
"ssh_wait_timeout": "360s"
},
{
- "type": "docker",
- "image": "ubuntu:16.04",
- "commit": true
- },
- {
- "type": "digitalocean",
- "api_token": "{{user `do_api_token`}}",
- "image": "{{user `distro`}}",
- "snapshot_name": "{{ user `distro` }}-{{ isotime \"200601021504\" }}",
- "region": "ams2",
- "size": "1gb"
+ "type": "openstack",
+ "ssh_username": "{{user `user`}}",
+ "ssh_password": "{{user `password`}}",
+ "image_name": "{{user `image_name`}}",
+ "floating_ip_pool": "{{user `os_floating_ip_pool`}}",
+ "reuse_ips": "true",
+ "source_image": "{{ user `os_image` }}",
+ "flavor": "{{ user `os_flavor` }}",
+ "user_data_file": "config-drive/user-data.yaml",
+ "tenant_id": "{{ user `os_tenant` }}",
+ "networks": ["{{ user `os_network` }}"],
+ "availability_zone": "{{user `os_zone`}}",
+ "identity_endpoint": "{{user `os_endpoint`}}",
+ "username": "{{user `os_username`}}",
+ "password": "{{user `os_password`}}"
}
]
}
diff --git a/ubuntu-16.04/template.json.env b/ubuntu-16.04/template.json.env
new file mode 100644
index 0000000..12be33e
--- /dev/null
+++ b/ubuntu-16.04/template.json.env
@@ -0,0 +1,19 @@
+# Env variables and description:
+DO_API_TOKEN
+
+# base.sh
+UBUNTU_BASEURL=http://mirror.mirantis.com/stable/ubuntu/
+
+# salt.sh
+SALTSTACK_REPO=${SALTSTACK_REPO:-"deb [arch=amd64] https://mirror.mirantis.com/stable/saltstack-2016.3/xenial/ xenial main"}
+SALTSTACK_GPG=${SALTSTACK_GPG:-"https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3/SALTSTACK-GPG-KEY.pub"}
+
+
+
+IMAGE_NAME # resulted openstack|qemu image name part
+
+# Openstack-related
+OS_USERNAME
+OS_PASSWORD
+OS_FLAVOR
+OS_IMAGE # base image