[TryMOS] Rename startup scripts and location
* Fix getting storage networks
* Make variables modular
* Drop hardcoded sleep in wait
Related-Prod: PRODX-3456
Change-Id: Ibd7aece3bd6262d2ad579d20e87d8af661cef3d9
diff --git a/de/heat-templates/scripts/instance_boot.sh b/de/heat-templates/scripts/instance_boot.sh
index e4405d0..be762bf 100644
--- a/de/heat-templates/scripts/instance_boot.sh
+++ b/de/heat-templates/scripts/instance_boot.sh
@@ -92,6 +92,7 @@
CONTROL_IP_ADDRESS=$(ip route get ${CONTROL_NETWORK_CIDR%/*} | head -n1 | fgrep -v ' via ' | awk '/ src / {print $6}')
else
CONTROL_IP_ADDRESS=$(get_interface_prefix ${DEFAULT_INTERFACE} | awk -F '/' '{print $1}')
+ CONTROL_NETWORK_CIDR=$(ip route show dev ${DEFAULT_INTERFACE} | awk '/kernel/ {print $1}')
fi
diff --git a/trymos/README b/trymos/README
index f9368d2..d33aba6 100644
--- a/trymos/README
+++ b/trymos/README
@@ -24,25 +24,16 @@
}
}
-5. Create an image from the snapshot in EC2 Service -> Elastic Block Store -> Snapshots -> Actions -> Create image
- with root storage 30Gb and additional volume 20Gb (volume type EBS (gp3))
+5. Wait unless task is completed `aws ec2 describe-import-snapshot-tasks --import-task-ids <task-id>`
-6. Launch instance from the image EC2 Service -> Images -> AMIs -> Launch with flavor minimal 16 CPUs and
+6. Create an image from the snapshot in EC2 Service -> Elastic Block Store -> Snapshots -> Actions -> Create image
+ with root storage 35Gb and additional volume 20Gb (volume type EBS (gp3))
+
+7. Launch instance from the image EC2 Service -> Images -> AMIs -> Launch with flavor minimal 16 CPUs and
30Gb RAM (c4.4xlarge).
-7. Connect to intance via external IP with keyfile which was defined during instance was brought up as root
+8. Connect to intance via external IP with keyfile which was defined during instance was brought up as root
ssh 13.59.177.99 -i ./trymos-pair.pem -l root
-8. Run following commands to prepare env for deployment
- sed -i 's/vdb/xvdb/g' /usr/share/metadata/ceph.yaml
-
- and export control network CIDR
- root@ip-172-31-11-89:/srv/bin# ip route
- default via 172.31.0.1 dev eth0 proto dhcp src 172.31.11.89 metric 100
- 172.31.0.0/20 dev eth0 proto kernel scope link src 172.31.11.89
- 172.31.0.1 dev eth0 proto dhcp scope link src 172.31.11.89 metric 100
-
- root@ip-172-31-11-89:/srv/bin# export CONTROL_NETWORK_CIDR='172.31.0.0/20'
-
9. Run installation script
- /srv/bin/bootstrap_trymos_aws.sh
+ /usr/share/trymos/launch.sh
diff --git a/trymos/heat-templates/env/aio.yaml b/trymos/heat-templates/env/aio.yaml
index 21c5226..7fc1f5c 100644
--- a/trymos/heat-templates/env/aio.yaml
+++ b/trymos/heat-templates/env/aio.yaml
@@ -3,7 +3,7 @@
"MCP2::SrvInstances": ../fragments/SrvInstancesVM.yaml
parameters:
- image: trymos-bionic-amd64-20210318-154934
+ image: trymos-bionic-amd64-6.12.0-20210322124800
public_net_id: public
cluster_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCp0evjOaK8c8SKYK4r2+0BN7g+8YSvQ2n8nFgOURCyvkJqOHi1qPGZmuN0CclYVdVuZiXbWw3VxRbSW3EH736VzgY1U0JmoTiSamzLHaWsXvEIW8VCi7boli539QJP0ikJiBaNAgZILyCrVPN+A6mfqtacs1KXdZ0zlMq1BPtFciR1JTCRcVs5vP2Wwz5QtY2jMIh3aiwkePjMTQPcfmh1TkOlxYu5IbQyZ3G1ahA0mNKI9a0dtF282av/F6pwB/N1R1nEZ/9VtcN2I1mf1NW/tTHEEcTzXYo1R/8K9vlqAN8QvvGLZtZduGviNVNoNWvoxaXxDt8CPv2B2NCdQFZp
private_floating_network_cidr: '10.11.12.0/24'
diff --git a/trymos/heat-templates/scripts/instance_boot.sh b/trymos/heat-templates/scripts/instance_boot.sh
index 5b3261c..fbb531f 100644
--- a/trymos/heat-templates/scripts/instance_boot.sh
+++ b/trymos/heat-templates/scripts/instance_boot.sh
@@ -32,7 +32,7 @@
fi
}
-bash /srv/bin/bootstrap_trymos_aws.sh
+bash /usr/share/trymos/launch.sh
if [[ "$?" == "0" ]]; then
wait_condition_send "SUCCESS" "Deploying TryMOS successfuly ."
diff --git a/trymos/image_build/build_image.sh b/trymos/image_build/build_image.sh
index e935332..e667636 100755
--- a/trymos/image_build/build_image.sh
+++ b/trymos/image_build/build_image.sh
@@ -1,7 +1,7 @@
#!/bin/bash
-export IMAGE_BASE_URL="https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img"
-export IMAGE_MD5_URL="https://cloud-images.ubuntu.com/bionic/current/MD5SUMS"
+export IMAGE_BASE_URL="http://images.mcp.mirantis.net/bionic-server-cloudimg-amd64-20190612.img"
+export IMAGE_MD5_URL="http://images.mcp.mirantis.net/bionic-server-cloudimg-amd64-20190612.img.md5"
export VM_NAME="trymos-bionic-amd64.qcow2"
export PACKER_LOG=1
diff --git a/trymos/image_build/files/usr/share/metadata/ceph.yaml b/trymos/image_build/files/usr/share/metadata/ceph.yaml
deleted file mode 100644
index 24f92f2..0000000
--- a/trymos/image_build/files/usr/share/metadata/ceph.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-storageDevices:
- - name: vdb
- role: hdd
- sizeGb: 2
-ramGb: 8
-cores: 2
diff --git a/trymos/image_build/files/usr/share/trymos/environment/amazon b/trymos/image_build/files/usr/share/trymos/environment/amazon
new file mode 100644
index 0000000..67324b4
--- /dev/null
+++ b/trymos/image_build/files/usr/share/trymos/environment/amazon
@@ -0,0 +1,8 @@
+export NTP_SERVERS=169.254.169.123
+export DOCKER_EE_URL=https://storebits.docker.com/ubuntu
+export DOCKER_UCP_VERSION=3.3.6
+export DOCKER_UCP_IMAGE=mirantis.azurecr.io/lcm/docker/ucp:${DOCKER_UCP_VERSION}
+export MIRANTIS_ARTIFACTORY_URL=https://binary.mirantis.com
+export CEPH_STORAGE_OSD_DEVICE="xvdb"
+export CEPH_CLUSTER_FILE="${RELEASE_OPENSTACK_K8S}/examples/miraceph/aio-ceph_local_drives_openstack.yaml"
+export OPENSTACK_CONTEXT_NAME="aio-core-ceph-local-non-dvr"
diff --git a/trymos/image_build/files/usr/share/trymos/environment/openstack b/trymos/image_build/files/usr/share/trymos/environment/openstack
new file mode 100644
index 0000000..373e7c6
--- /dev/null
+++ b/trymos/image_build/files/usr/share/trymos/environment/openstack
@@ -0,0 +1,4 @@
+export CEPH_STORAGE_OSD_DEVICE="vdb"
+export CEPH_CLUSTER_FILE="${RELEASE_OPENSTACK_K8S}/examples/miraceph/aio-ceph_local_drives_openstack.yaml"
+export OPENSTACK_CONTEXT_NAME="aio-core-ceph-local-non-dvr"
+export MIRANTIS_ARTIFACTORY_URL=https://binary.mirantis.com
diff --git a/trymos/image_build/files/srv/bin/bootstrap_trymos_aws.sh b/trymos/image_build/files/usr/share/trymos/launch.sh
similarity index 67%
rename from trymos/image_build/files/srv/bin/bootstrap_trymos_aws.sh
rename to trymos/image_build/files/usr/share/trymos/launch.sh
index 659551e..12eb105 100755
--- a/trymos/image_build/files/srv/bin/bootstrap_trymos_aws.sh
+++ b/trymos/image_build/files/usr/share/trymos/launch.sh
@@ -9,28 +9,41 @@
export NODE_TYPE='trymos'
export NODE_METADATA='{"labels": {"local-volume-provisioner": "enabled", "openstack-compute-node": "enabled", "openstack-control-plane": "enabled", "openstack-gateway": "enabled", "openvswitch": "enabled", "role": "ceph-osd-node"}}'
+_ENVIRONMENT_FOLDER="/usr/share/trymos/environment"
+_DEPLOY_FINISHED_LOCK="/var/tmp/trymos-deploy-finished.lock"
+_TRYMOS_INFO_FILE="/etc/trymos-info"
+
+
+export RELEASE_OPENSTACK_K8S=/srv/release-openstack-k8s
+export CERT_REPO_DIR=/srv/certs
+export CSR_CONF="${CERT_REPO_DIR}/api-csr.yaml"
+export SSL_BARE_NAME='api'
+
+function get_instance_type {
+ if grep "OpenStack" -i /sys/devices/virtual/dmi/id/product_name -q; then
+ echo "openstack"
+ elif grep "amazon" -i /sys/devices/virtual/dmi/id/product_version -q; then
+ echo "amazon"
+ else
+ echo "UNKNOWN"
+ fi
+}
+export _INSTANCE_TYPE=$(get_instance_type)
+
+if [[ -f "$_ENVIRONMENT_FOLDER/${_INSTANCE_TYPE}" ]]; then
+ source $_ENVIRONMENT_FOLDER/${_INSTANCE_TYPE}
+else
+ echo "Unknown instance type $_INSTANCE_TYPE"
+ exit 1
+fi
+
+source /usr/share/trymos/functions
export STORAGE_CLUSTER_CIDR=${CONTROL_NETWORK_CIDR}
export STORAGE_PUBLIC_CIDR=${CONTROL_NETWORK_CIDR}
-export NTP_SERVERS=${NTP_SERVERS:-169.254.169.123}
-
-#TryMOS variables
-export DOCKER_UCP_VERSION=3.3.3
-export DOCKER_UCP_IMAGE=${DOCKER_UCP_IMAGE:-mirantis.azurecr.io/lcm/docker/ucp:${DOCKER_UCP_VERSION}}
-export RELEASE_OPENSTACK_K8S=${RELEASE_OPENSTACK_K8S:-/srv/release-openstack-k8s}
-export CERT_REPO_DIR=${CERT_REPO_DIR:-/srv/certs}
-export CSR_CONF="${CERT_REPO_DIR}/api-csr.yaml"
-export SSL_BARE_NAME='api'
-export MIRANTIS_ARTIFACTORY_URL=${MIRANTIS_ARTIFACTORY_URL:-https://binary.mirantis.com}
-export CEPH_CLUSTER_FILE="${RELEASE_OPENSTACK_K8S}/examples/miraceph/aio-ceph_local_drives_openstack.yaml"
-export OPENSTACK_CONTEXT_NAME=${OPENSTACK_CONTEXT_NAME:-"aio-core-ceph-local-non-dvr"}
export TUNNEL_INTERFACE=${TUNNEL_INTERFACE:-${DEFAULT_INTERFACE}}
export LIVE_MIGRATION_INTERFACE=${LIVE_MIGRATION_INTERFACE:-${DEFAULT_INTERFACE}}
-_DEPLOY_FINISHED_LOCK="/var/tmp/trymos-deploy-finished.lock"
-
-source /usr/share/trymos/functions
-
if [[ -f ${_DEPLOY_FINISHED_LOCK} ]]; then
echo "TryMOS deploy has been performed on this node."
echo "If you want to redeploy please delete lock file"
@@ -43,6 +56,19 @@
echo "$msg"
}
+function generate_ceph_metadata {
+ mkdir -p /usr/share/metadata
+ cat << EOF > /usr/share/metadata/ceph.yaml
+storageDevices:
+ - name: ${CEPH_STORAGE_OSD_DEVICE}
+ role: hdd
+ sizeGb: 2
+ramGb: 8
+cores: 2
+EOF
+
+}
+
function configure_virt_public_iface {
@@ -139,6 +165,59 @@
fi
}
+function wait_for_pods {
+ local namespace=$1
+ local component_filter=${2:-""}
+ local timeout=${3:-900}
+ local delay=${4:-30}
+
+
+ end=$(date +%s)
+ end=$((end + timeout))
+ while true; do
+ sleep $delay
+ if kubectl get pods --namespace=${namespace} $component_filter 2>&1 |grep -q 'No resources found'; then
+ continue
+ fi
+
+ kubectl get pods --namespace=${namespace} $component_filter -o json | jq -r \
+ '.items[].status.phase' | grep Pending > /dev/null && \
+ PENDING="True" || PENDING="False"
+
+ query='.items[]|select(.status.phase=="Running")'
+ query="$query|.status.containerStatuses[].ready"
+ kubectl get pods --namespace=${namespace} $component_filter -o json | jq -r "$query" | \
+ grep false > /dev/null && READY="False" || READY="True"
+
+ kubectl get jobs --namespace=${namespace} $component_filter -o json | jq -r \
+ '.items[] | .spec.completions == .status.succeeded' | \
+ grep false > /dev/null && JOBR="False" || JOBR="True"
+ [ $PENDING == "False" -a $READY == "True" -a $JOBR == "True" ] && \
+ break || true
+ sleep 5
+ now=$(date +%s)
+ if [ $now -gt $end ] ; then
+ echo "Containers failed to start after $timeout seconds"
+ echo
+ kubectl get pods --namespace ${namespace} $component_filter -o wide
+ echo
+ if [ $PENDING == "True" ] ; then
+ echo "Some pods are in pending state:"
+ kubectl get pods $component_filter --field-selector=status.phase=Pending -n ${namespace} -o wide
+ fi
+ [ $READY == "False" ] && echo "Some pods are not ready"
+ [ $JOBR == "False" ] && echo "Some jobs have not succeeded"
+ exit -1
+ fi
+ done
+}
+
+function wait_for_os_component {
+ local component=$1
+ echo "Waiting for: $component"
+ wait_for_pods openstack "-l application=$component"
+ echo "All resources for component $component Ready"
+}
function install_release_controllers {
@@ -148,7 +227,6 @@
kubectl apply -f ${release_object}
}
retry 3 "Can't apply ${release_object}" apply_retry
-# sleep 5
done
}
@@ -160,7 +238,9 @@
info "Preparing DNS and SSL configuration."
kubectl apply -f ${RELEASE_OPENSTACK_K8S}/release/ci/30-coredns.yaml
- sleep 300
+
+ wait_for_pods coredns "" 300
+
EXTERNAL_DNS_IP=$(kubectl -n coredns get services coredns-coredns -o jsonpath='{.status.loadBalancer.ingress[].ip}')
if [ -z ${EXTERNAL_DNS_IP} ]; then
EXTERNAL_DNS_IP=$(kubectl -n coredns get services coredns-coredns -o jsonpath='{.spec.clusterIP}')
@@ -227,7 +307,13 @@
yq w -i ${RELEASE_OPENSTACK_K8S}/examples/osdpl/${OPENSTACK_CONTEXT_NAME}.yaml "spec.services.compute.nova.values.conf.nova.libvirt.virt_type" "qemu"
kubectl apply -f ${RELEASE_OPENSTACK_K8S}/examples/osdpl/${OPENSTACK_CONTEXT_NAME}.yaml
- sleep 1800
+
+ wait_for_pods openstack "-l app.kubernetes.io/name=cache" 1200
+
+ for component in mariadb rabbitmq memcached keystone glance cinder nova neutron barbican octavia cinder designate heat libvirt openvswitch; do
+ wait_for_os_component $component
+ done
+ wait_for_pods openstack
kubectl -n openstack create job --from=cronjob/nova-cell-setup nova-cell-setup-pd01-$(cat /dev/urandom | tr -dc '[a-z]' | head -c3)
info "Openstack was deployed successfully..."
@@ -256,10 +342,58 @@
}
function deploy_finished {
- touch ${DEPLOY_FINISHED_LOCK}
+ touch ${_DEPLOY_FINISHED_LOCK}
}
+function write_trymos_info {
+ local distro_info
+ local os_controller_version
+ local ceph_controller_version
+ local os_version
+ local keystone_client_pod
+ local os_admin_username
+ local os_admin_password
+ distro_info=$(lsb_release -sd)
+ os_controller_version=$(kubectl -n osh-system get helmbundles openstack-operator -o jsonpath='{.status.releaseStatuses.openstack-operator.version}')
+ ceph_controller_version=$(kubectl -n osh-system get helmbundles ceph-operator -o jsonpath='{.status.releaseStatuses.ceph-operator.version}')
+ os_version=$(kubectl -n openstack get osdpl osh-dev -o jsonpath='{.spec.openstack_version}')
+ keystone_client_pod=$(kubectl -n openstack get pods -l application=keystone,component=client -o jsonpath='{.items[*].metadata.name}')
+ os_admin_username=$(kubectl -n openstack exec -it $keystone_client_pod -- bash -c "echo \$OS_USERNAME")
+ os_admin_password=$(kubectl -n openstack exec -it $keystone_client_pod -- bash -c "echo \$OS_PASSWORD")
+
+ cat << EOF > ${_TRYMOS_INFO_FILE}
+####################################################
+#################### TryMOS ########################
+####################################################
+
+############### Components Vresions ################
+
+OS: ${distro_info}
+OpenStack Controller: ${os_controller_version}
+Ceph Controller: ${ceph_controller_version}
+OpenStack Version: ${os_version}
+
+############# UCP access information ###############
+UCP Admin Username: ${UCP_USERNAME}
+UCP Admin Password: ${UCP_PASSWORD}
+
+########### OpenStack access information ###########
+OpenStack Admin Username: ${os_admin_username}
+OpenStack Admin Password: ${os_admin_password}
+
+####################################################
+EOF
+
+ cat <<EOF > /etc/update-motd.d/99-trymos-info
+#!/bin/sh
+cat ${_TRYMOS_INFO_FILE}
+EOF
+
+ chmod +x /etc/update-motd.d/99-trymos-info
+
+ cat ${_TRYMOS_INFO_FILE}
+}
case "$NODE_TYPE" in
# Please keep the "prepare_metadata_files", "disable-rp-filter", "network_config" and "prepare_network" functions
@@ -284,6 +418,7 @@
install_kubectl
wait_for_node
set_node_labels
+ generate_ceph_metadata
collect_ceph_metadata
configure_contrack
disable_iptables_for_bridges
@@ -295,6 +430,7 @@
install_ceph
install_openstack
configure_public_resolve
+ write_trymos_info
deploy_finished
;;
*)
diff --git a/trymos/image_build/template.json b/trymos/image_build/template.json
index 9af97a3..d889b97 100644
--- a/trymos/image_build/template.json
+++ b/trymos/image_build/template.json
@@ -10,15 +10,6 @@
},
"provisioners": [
{
- "type": "shell",
- "inline": ["mkdir -p /usr/share/trymos/"]
- },
- {
- "type": "file",
- "source": "../../de/heat-templates/scripts/instance_boot.sh",
- "destination": "/usr/share/trymos/functions"
- },
- {
"type": "file",
"source": "./certs",
"destination": "/srv/"
@@ -30,13 +21,13 @@
},
{
"type": "file",
- "source": "files/srv/",
- "destination": "/srv"
+ "source": "files/usr/share",
+ "destination": "/usr"
},
{
"type": "file",
- "source": "files/usr/share",
- "destination": "/usr"
+ "source": "../../de/heat-templates/scripts/instance_boot.sh",
+ "destination": "/usr/share/trymos/functions"
},
{
"type": "file",