Update BM deploy
- Use bootstrapv2
- Update templates according to latest release
PROD-37312
Change-Id: Ib4e60febbd8a4c04b5061928b25bdaaeee701028
diff --git a/bm_mcc_mosk/kaas-mgmt/cluster.yaml b/bm_mcc_mosk/kaas-mgmt/cluster.yaml
index 5e39da4..1c8f0f0 100644
--- a/bm_mcc_mosk/kaas-mgmt/cluster.yaml
+++ b/bm_mcc_mosk/kaas-mgmt/cluster.yaml
@@ -2,28 +2,27 @@
apiVersion: cluster.k8s.io/v1alpha1
kind: Cluster
metadata:
+ name: kaas-mgmt
labels:
kaas.mirantis.com/provider: baremetal
- name: kaas-mgmt
namespace: default
spec:
clusterNetwork:
- pods:
- cidrBlocks:
- - 10.200.0.0/16
services:
cidrBlocks:
- - 10.201.0.0/16
+ - 10.201.0.0/16
+ pods:
+ cidrBlocks:
+ - 10.200.0.0/16
providerSpec:
value:
- kind: BaremetalClusterProviderSpec
- #release: mke-16-1-0-3-7-5
apiVersion: baremetal.k8s.io/v1alpha1
- dedicatedControlPlane: false
- dedicatedMetallbPools: true
- loadBalancerHost: ""
+ kind: BaremetalClusterProviderSpec
+ release: mke-16-3-0-3-7-12
dnsNameservers:
- 172.18.176.6
+ dedicatedControlPlane: false
+ dedicatedMetallbPools: true
helmReleases:
- name: metallb
values: {}
@@ -37,11 +36,10 @@
prometheusServer:
persistentVolumeClaimSize: 16Gi
kaas:
- #release: kaas-2-26-0
- management:
- enabled: true
+ release: kaas-2-28-0
regional:
- - helmReleases:
+ - provider: baremetal
+ helmReleases:
- name: baremetal-operator
values: {}
- name: baremetal-provider
@@ -51,4 +49,5 @@
dynamic_bootp: true
- name: kaas-ipam
values: {}
- provider: baremetal
+ management:
+ enabled: true
diff --git a/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml b/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml
index 2a88aa8..73234f7 100644
--- a/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml
+++ b/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml
@@ -70,23 +70,6 @@
- 172.16.181.51-172.16.181.100
---
-apiVersion: ipam.mirantis.com/v1alpha1
-kind: MetalLBConfigTemplate
-metadata:
- labels:
- kaas.mirantis.com/provider: baremetal
- cluster.sigs.k8s.io/cluster-name: kaas-mgmt
- name: kaas-mgmt-metallb
- namespace: default
-spec:
- templates:
- l2Advertisements: |
- - name: default
- spec:
- ipAddressPools:
- - default
- - services-pxe
----
# L2Template allows you to create advanced host networking
# configurations for your management cluster. For example,
# you can create bond interfaces on top of physical interfaces
diff --git a/bm_mcc_mosk/kaas-mgmt/serviceuser.yaml b/bm_mcc_mosk/kaas-mgmt/serviceuser.yaml
new file mode 100644
index 0000000..a709102
--- /dev/null
+++ b/bm_mcc_mosk/kaas-mgmt/serviceuser.yaml
@@ -0,0 +1,10 @@
+apiVersion: kaas.mirantis.com/v1alpha1
+kind: ServiceUserList
+items:
+- apiVersion: kaas.mirantis.com/v1alpha1
+ kind: ServiceUser
+ metadata:
+ name: admin
+ spec:
+ password:
+ value: admin
\ No newline at end of file
diff --git a/bm_mcc_mosk/mcc_seed_2401_userdata.yaml b/bm_mcc_mosk/mcc_seed_2401_userdata.yaml
index 17292d1..4a17192 100644
--- a/bm_mcc_mosk/mcc_seed_2401_userdata.yaml
+++ b/bm_mcc_mosk/mcc_seed_2401_userdata.yaml
@@ -108,12 +108,7 @@
sudo chown -R regional:regional /home/regional
apt-get ${APT_OPTS} -y install bridge-utils docker.io ipmitool wget golang-cfssl jq
usermod -aG docker ubuntu
- cd /root/
- wget https://binary.mirantis.com/releases/get_container_cloud.sh
- chmod 0755 get_container_cloud.sh
- ./get_container_cloud.sh
- cp /root/mirantis.lic kaas-bootstrap/
-
+
echo "Sending wait_condition signal"
wait_condition_send "SUCCESS" "Instance is UP and running"
@@ -170,57 +165,10 @@
parameters:
forward-delay: 4
stp: false
- - path: /root/do_deploy_mcc_mgmt.sh
- content: |
- set -e
- source /root/env_vars.sh
- while [ ! -d "${OUT_DIR}" ] ; do sleep 4 ; done
- [[ "$(sed -n 693p /root/kaas-bootstrap/bootstrap.sh)" -eq "configure" ]] && sed -i "693d" /root/kaas-bootstrap/bootstrap.sh
- # PROD-44779
- set +e -o pipefail
- /root/kaas-bootstrap/bootstrap.sh all 2>&1 | tee deploy_mcc_mgmt_output.log
- retcode=$?
- set -e
- if [[ $retcode -ne 0 ]] ; then
- grep "Keycloak service is unavailable" deploy_mcc_mgmt_output.log && echo "PRODX-44779 is still there, ignoring" || exit $retcode
- else
- echo 'REMOVE PRODX-44779 WORKAROUND'
- fi
- export KUBECONFIG=/root/kubeconfig
- echo r00tme | /root/kaas-bootstrap/container-cloud bootstrap user add --username root --roles global-admin,management-admin,reader,writer,operator --kubeconfig kubeconfig --password-stdin
- - path: /root/do_deploy_child.sh
- content: |
- set -e
- while [ ! -d /root/bm_mcc_mosk/child/cluster ] ; do echo 'no child templates!' ; sleep 4 ; done
- export KUBECONFIG=/root/kubeconfig
- /root/kaas-bootstrap/bin/kubectl apply -f /root/bm_mcc_mosk/child/cluster/project.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/ssh_pubkeys.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/cluster.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/subnets.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/l2_templates.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/metallbconfig.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/machines.yaml
- sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/kaascephcluster.yaml
-
- path: /etc/udev/rules.d/60-ssd-scheduler.rules
content: |
ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="deadline"
- - path: /root/env_vars.sh
- content: |
- export KAAS_BM_ENABLED="true"
- export KAAS_BM_PXE_IP="172.16.180.5"
- export KAAS_BM_PXE_MASK="23"
- export KAAS_BM_PXE_BRIDGE="br0"
- export CLUSTER_NAME=kaas-mgmt
- export OUT_DIR=/root/bm_mcc_mosk/kaas-mgmt/
- export KAAS_BOOTSTRAP_INFINITE_TIMEOUT=true
- - path: /root/get_child_kubeconfig.sh
- content: |
- /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get secrets mosk-kubeconfig -o jsonpath='{.data.admin\.conf}' | base64 -d | sed 's/:5443/:443/g' > /root/child.kubeconfig
- export KUBECONFIG=/root/child.kubeconfig
- path: /root/mirantis.lic
content: |
eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9Cg.eyJleHAiOjE3MzgzOTY4MDAsImlhdCI6MTY3NTMyNDgwMCwic3ViIjoiZGV2fHNpLWRldiIsImxpY2Vuc2UiOnsiZGV2Ijp0cnVlLCJsaW1pdHMiOnsiY2x1c3RlcnMiOjAsIndvcmtlcnNfcGVyX2NsdXN0ZXIiOjB9LCJvcGVuc3RhY2siOnsiY2x1c3RlcnMiOjAsIndvcmtlcnNfcGVyX2NsdXN0ZXIiOjB9fX0K.18naIn5bHkrQJGnqsiv8BHAEhdz_mnMSR2Oz0hAKyhVTdn5Hd7ESJFvPe2agEl7IJf4n6--NPa9zqW0y9zcixnoxB_7xvMntNCaPzfAap8Lm7RSghDJicyJ1xXTj4NNf3ocnbA8rCUNkrSbh2GKFNBqiDMqZTGC7Jozee5HjBzaxFUF0Z0Nr3T0q53DrZmiAhe0P8LtbxFhMICptcMnX-c4mw_hc5TziLZdpR0TUCJk4B0Cit4PABzZWjDCt5gWpy70ZCTTG2xo5dikd-WYBp6f43U5LUroYkhKTHjLMphHnsEDDBu2qaV18ONSuFSQ-Sfg_Mg9ndS_IMTvS9IipsA
diff --git a/bm_mcc_mosk/seed/do_deploy_child.sh b/bm_mcc_mosk/seed/do_deploy_child.sh
new file mode 100644
index 0000000..6229555
--- /dev/null
+++ b/bm_mcc_mosk/seed/do_deploy_child.sh
@@ -0,0 +1,13 @@
+set -e
+while [ ! -d /root/bm_mcc_mosk/child/cluster ] ; do echo 'no child templates!' ; sleep 4 ; done
+export KUBECONFIG=/root/kubeconfig
+/root/kaas-bootstrap/bin/kubectl apply -f /root/bm_mcc_mosk/child/cluster/project.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/ssh_pubkeys.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/cluster.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/subnets.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/l2_templates.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/metallbconfig.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/machines.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/kaascephcluster.yaml
diff --git a/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh b/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
new file mode 100644
index 0000000..b14994c
--- /dev/null
+++ b/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
@@ -0,0 +1,77 @@
+set -e
+source /root/bm_mcc_mosk/seed/env_vars.sh
+
+pushd /root/
+ wget https://binary.mirantis.com/releases/get_container_cloud.sh
+ chmod 0755 get_container_cloud.sh
+ ./get_container_cloud.sh
+ cp /root/mirantis.lic kaas-bootstrap/
+popd
+
+/root/kaas-bootstrap/bootstrap.sh bootstrapv2
+
+
+export KUBECONFIG=/root/.kube/kind-config-clusterapi
+pushd /root/bm_mcc_mosk/kaas-mgmt
+ /root/kaas-bootstrap/bin/kubectl apply -f bootstrapregion.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f serviceuser.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f cluster.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f baremetalhostprofiles.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f ipam-objects.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f baremetalhosts.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f metallbconfig.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f machines.yaml && sleep 2
+popd
+
+retry=60
+interval=60
+
+counter=0
+while (( counter++ < retry )) ; do
+ sleep $interval
+ avail_count=$(/root/kaas-bootstrap/bin/kubectl get bmh -o go-template='{{- range .items -}} {{.status.provisioning.state}}{{"\n"}} {{- end -}}' | (grep -c available || /bin/true))
+ echo "loop=${counter}, available state=${avail_count}"
+ if (( avail_count == 3 )); then
+ break
+ fi
+done
+if (( counter > retry )) ; then
+ echo "Not all nodes are available after timeout"
+ exit 1
+fi
+
+counter=0
+while (( counter++ < retry )) ; do
+ sleep $interval
+ status=$(/root/kaas-bootstrap/bin/kubectl get bootstrapregions -o go-template='{{(index .items 0).status.ready}}{{"\n"}}')
+ echo "loop=${counter}, status=${status}"
+ if [[ "$status" == "true" ]]; then
+ break
+ fi
+done
+if (( counter > retry )) ; then
+ echo "BootstrapRegion timed out"
+ exit 1
+fi
+
+/root/kaas-bootstrap/container-cloud bootstrap approve all
+
+counter=0
+retry=90
+while (( counter++ < retry )) ; do
+ sleep $interval
+ status=$(/root/kaas-bootstrap/bin/kubectl get cluster kaas-mgmt -o go-template='{{.status.providerStatus.ready}}{{"\n"}}')
+ echo "loop=${counter}, status=${status}"
+ if [[ "$status" == "true" ]]; then
+ break
+ fi
+done
+if (( counter > retry )) ; then
+ echo "MGMT Cluster deployment timed out"
+ exit 1
+fi
+cd
+/root/kaas-bootstrap/container-cloud get cluster-kubeconfig --cluster-name kaas-mgmt --kubeconfig-output /root/kubeconfig
+echo r00tme | /root/kaas-bootstrap/container-cloud bootstrap user add --username root --roles global-admin,management-admin,reader,writer,operator --kubeconfig kubeconfig --password-stdin
+/root/kaas-bootstrap/bin/kind delete cluster -n clusterapi
+unset KUBECONFIG
diff --git a/bm_mcc_mosk/seed/env_vars.sh b/bm_mcc_mosk/seed/env_vars.sh
new file mode 100644
index 0000000..6252ad9
--- /dev/null
+++ b/bm_mcc_mosk/seed/env_vars.sh
@@ -0,0 +1,7 @@
+export KAAS_BM_ENABLED="true"
+export KAAS_BM_PXE_IP="172.16.180.5"
+export KAAS_BM_PXE_MASK="23"
+export KAAS_BM_PXE_BRIDGE="br0"
+export CLUSTER_NAME=kaas-mgmt
+export OUT_DIR=/root/bm_mcc_mosk/kaas-mgmt/
+#export KAAS_BOOTSTRAP_INFINITE_TIMEOUT=true
\ No newline at end of file
diff --git a/bm_mcc_mosk/seed/get_child_kubeconfig.sh b/bm_mcc_mosk/seed/get_child_kubeconfig.sh
new file mode 100644
index 0000000..23565e7
--- /dev/null
+++ b/bm_mcc_mosk/seed/get_child_kubeconfig.sh
@@ -0,0 +1,2 @@
+/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get secrets mosk-kubeconfig -o jsonpath='{.data.admin\.conf}' | base64 -d | sed 's/:5443/:443/g' > /root/child.kubeconfig
+export KUBECONFIG=/root/child.kubeconfig
\ No newline at end of file
diff --git a/jobs/pipelines/mosk/deploy-bm.groovy b/jobs/pipelines/mosk/deploy-bm.groovy
index a2cae19..ed4f920 100644
--- a/jobs/pipelines/mosk/deploy-bm.groovy
+++ b/jobs/pipelines/mosk/deploy-bm.groovy
@@ -59,8 +59,8 @@
stage("Bootstrap MCC+child"){
sh "scp ${ssh_params} ${MAINTENANCE_TEAM_SSH_KEY} root@${seed_ext_ip}:/root/.ssh/id_rsa "
sh "scp -r ${ssh_params} bm_mcc_mosk root@${seed_ext_ip}: "
- sh "ssh ${ssh_params} root@${seed_ext_ip} bash do_deploy_mcc_mgmt.sh"
- sh "ssh ${ssh_params} root@${seed_ext_ip} bash do_deploy_child.sh"
+ sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh"
+ sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/do_deploy_child.sh"
// TODO: unhardcode cluster names
check_child_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get cluster mosk -o jsonpath='{.status.providerStatus.ready}' 2>/dev/null || echo 'ssh error, ignoring'"
check_kcc_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get kcc mosk-ceph -o jsonpath='{.status.shortClusterInfo.state}' 2>/dev/null || echo 'ssh error, ignoring'"
@@ -84,7 +84,7 @@
} //while
} //stage MCC
stage("Prepare and deploy MOSK"){
- sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/get_child_kubeconfig.sh'"
+ sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/bm_mcc_mosk/seed/get_child_kubeconfig.sh'"
sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/bm_mcc_mosk/utils/tsl_gen.sh'"
kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/osdpl-ssl-secrets.yaml"
diff --git a/jobs/templates/bm-mcc-mosk-base.yml b/jobs/templates/bm-mcc-mosk-base.yml
index 90b8980..e68aaab 100644
--- a/jobs/templates/bm-mcc-mosk-base.yml
+++ b/jobs/templates/bm-mcc-mosk-base.yml
@@ -1,7 +1,7 @@
- job-template:
project-type: pipeline
description: '{job-description}'
- concurrent: true
+ concurrent: false
disabled: false
name: bm-mcc-mosk-base
parameters:
@@ -50,7 +50,7 @@
name: EXTRA_VARS
trim: 'false'
- bool:
- default: true
+ default: false
description: Run tempest after deploy
name: RUN_TESTS_AFTER_DEPLOY
pipeline-scm: