Update BM deploy

- Use bootstrapv2
- Update templates according to latest release

PROD-37312

Change-Id: Ib4e60febbd8a4c04b5061928b25bdaaeee701028
diff --git a/bm_mcc_mosk/kaas-mgmt/cluster.yaml b/bm_mcc_mosk/kaas-mgmt/cluster.yaml
index 5e39da4..1c8f0f0 100644
--- a/bm_mcc_mosk/kaas-mgmt/cluster.yaml
+++ b/bm_mcc_mosk/kaas-mgmt/cluster.yaml
@@ -2,28 +2,27 @@
 apiVersion: cluster.k8s.io/v1alpha1
 kind: Cluster
 metadata:
+  name: kaas-mgmt
   labels:
     kaas.mirantis.com/provider: baremetal
-  name: kaas-mgmt
   namespace: default
 spec:
   clusterNetwork:
-    pods:
-      cidrBlocks:
-      - 10.200.0.0/16
     services:
       cidrBlocks:
-      - 10.201.0.0/16
+        - 10.201.0.0/16
+    pods:
+      cidrBlocks:
+        - 10.200.0.0/16
   providerSpec:
     value:
-      kind: BaremetalClusterProviderSpec
-      #release: mke-16-1-0-3-7-5
       apiVersion: baremetal.k8s.io/v1alpha1
-      dedicatedControlPlane: false
-      dedicatedMetallbPools: true
-      loadBalancerHost: ""
+      kind: BaremetalClusterProviderSpec
+      release: mke-16-3-0-3-7-12
       dnsNameservers:
       - 172.18.176.6
+      dedicatedControlPlane: false
+      dedicatedMetallbPools: true
       helmReleases:
       - name: metallb
         values: {}
@@ -37,11 +36,10 @@
           prometheusServer:
             persistentVolumeClaimSize: 16Gi
       kaas:
-        #release: kaas-2-26-0
-        management:
-          enabled: true
+        release: kaas-2-28-0
         regional:
-        - helmReleases:
+        - provider: baremetal
+          helmReleases:
           - name: baremetal-operator
             values: {}
           - name: baremetal-provider
@@ -51,4 +49,5 @@
                 dynamic_bootp: true
           - name: kaas-ipam
             values: {}
-          provider: baremetal
+        management:
+          enabled: true
diff --git a/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml b/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml
index 2a88aa8..73234f7 100644
--- a/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml
+++ b/bm_mcc_mosk/kaas-mgmt/ipam-objects.yaml
@@ -70,23 +70,6 @@
     - 172.16.181.51-172.16.181.100
 
 ---
-apiVersion: ipam.mirantis.com/v1alpha1
-kind: MetalLBConfigTemplate
-metadata:
-  labels:
-    kaas.mirantis.com/provider: baremetal
-    cluster.sigs.k8s.io/cluster-name: kaas-mgmt
-  name: kaas-mgmt-metallb
-  namespace: default
-spec:
-  templates:
-    l2Advertisements: |
-      - name: default
-        spec:
-          ipAddressPools:
-            - default
-            - services-pxe
----
 # L2Template allows you to create advanced host networking
 # configurations for your management cluster. For example,
 # you can create bond interfaces on top of physical interfaces
diff --git a/bm_mcc_mosk/kaas-mgmt/serviceuser.yaml b/bm_mcc_mosk/kaas-mgmt/serviceuser.yaml
new file mode 100644
index 0000000..a709102
--- /dev/null
+++ b/bm_mcc_mosk/kaas-mgmt/serviceuser.yaml
@@ -0,0 +1,10 @@
+apiVersion: kaas.mirantis.com/v1alpha1
+kind: ServiceUserList
+items:
+- apiVersion: kaas.mirantis.com/v1alpha1
+  kind: ServiceUser
+  metadata:
+    name: admin
+  spec:
+    password:
+      value: admin
\ No newline at end of file
diff --git a/bm_mcc_mosk/mcc_seed_2401_userdata.yaml b/bm_mcc_mosk/mcc_seed_2401_userdata.yaml
index 17292d1..4a17192 100644
--- a/bm_mcc_mosk/mcc_seed_2401_userdata.yaml
+++ b/bm_mcc_mosk/mcc_seed_2401_userdata.yaml
@@ -108,12 +108,7 @@
     sudo chown -R regional:regional /home/regional
     apt-get ${APT_OPTS} -y install bridge-utils docker.io ipmitool wget golang-cfssl jq
     usermod -aG docker ubuntu
-    cd /root/
-    wget https://binary.mirantis.com/releases/get_container_cloud.sh
-    chmod 0755 get_container_cloud.sh
-    ./get_container_cloud.sh
-    cp /root/mirantis.lic kaas-bootstrap/
-
+    
     echo "Sending wait_condition signal"
     wait_condition_send "SUCCESS" "Instance is UP and running"
 
@@ -170,57 +165,10 @@
             parameters:
                 forward-delay: 4
                 stp: false
-  - path: /root/do_deploy_mcc_mgmt.sh
-    content: |
-      set -e
-      source /root/env_vars.sh
-      while [ ! -d "${OUT_DIR}" ] ; do sleep 4 ; done
-      [[ "$(sed -n 693p /root/kaas-bootstrap/bootstrap.sh)" -eq "configure" ]] && sed -i "693d" /root/kaas-bootstrap/bootstrap.sh
-      # PROD-44779
-      set +e -o pipefail
-      /root/kaas-bootstrap/bootstrap.sh all 2>&1 | tee deploy_mcc_mgmt_output.log
-      retcode=$?
-      set -e
-      if [[ $retcode -ne 0 ]] ; then
-        grep "Keycloak service is unavailable" deploy_mcc_mgmt_output.log && echo "PRODX-44779 is still there, ignoring" || exit $retcode
-      else
-        echo 'REMOVE PRODX-44779 WORKAROUND'
-      fi
-      export KUBECONFIG=/root/kubeconfig
-      echo r00tme | /root/kaas-bootstrap/container-cloud bootstrap user add --username root --roles global-admin,management-admin,reader,writer,operator --kubeconfig kubeconfig --password-stdin
-  - path: /root/do_deploy_child.sh
-    content: |
-      set -e
-      while [ ! -d /root/bm_mcc_mosk/child/cluster ] ; do echo 'no child templates!' ; sleep 4 ; done
-      export KUBECONFIG=/root/kubeconfig
-      /root/kaas-bootstrap/bin/kubectl apply -f /root/bm_mcc_mosk/child/cluster/project.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/ssh_pubkeys.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/cluster.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/subnets.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/l2_templates.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/metallbconfig.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/machines.yaml
-      sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/kaascephcluster.yaml
-
   - path: /etc/udev/rules.d/60-ssd-scheduler.rules
     content: |
         ACTION=="add|change", KERNEL=="sd[a-z]", ATTR{queue/rotational}=="0", ATTR{queue/scheduler}="deadline"
 
-  - path: /root/env_vars.sh
-    content: |
-        export KAAS_BM_ENABLED="true"
-        export KAAS_BM_PXE_IP="172.16.180.5"
-        export KAAS_BM_PXE_MASK="23"
-        export KAAS_BM_PXE_BRIDGE="br0"
-        export CLUSTER_NAME=kaas-mgmt
-        export OUT_DIR=/root/bm_mcc_mosk/kaas-mgmt/
-        export KAAS_BOOTSTRAP_INFINITE_TIMEOUT=true
-  - path: /root/get_child_kubeconfig.sh
-    content: |
-        /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get secrets mosk-kubeconfig -o jsonpath='{.data.admin\.conf}' | base64 -d | sed 's/:5443/:443/g' > /root/child.kubeconfig
-        export KUBECONFIG=/root/child.kubeconfig
   - path: /root/mirantis.lic
     content: |
         eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9Cg.eyJleHAiOjE3MzgzOTY4MDAsImlhdCI6MTY3NTMyNDgwMCwic3ViIjoiZGV2fHNpLWRldiIsImxpY2Vuc2UiOnsiZGV2Ijp0cnVlLCJsaW1pdHMiOnsiY2x1c3RlcnMiOjAsIndvcmtlcnNfcGVyX2NsdXN0ZXIiOjB9LCJvcGVuc3RhY2siOnsiY2x1c3RlcnMiOjAsIndvcmtlcnNfcGVyX2NsdXN0ZXIiOjB9fX0K.18naIn5bHkrQJGnqsiv8BHAEhdz_mnMSR2Oz0hAKyhVTdn5Hd7ESJFvPe2agEl7IJf4n6--NPa9zqW0y9zcixnoxB_7xvMntNCaPzfAap8Lm7RSghDJicyJ1xXTj4NNf3ocnbA8rCUNkrSbh2GKFNBqiDMqZTGC7Jozee5HjBzaxFUF0Z0Nr3T0q53DrZmiAhe0P8LtbxFhMICptcMnX-c4mw_hc5TziLZdpR0TUCJk4B0Cit4PABzZWjDCt5gWpy70ZCTTG2xo5dikd-WYBp6f43U5LUroYkhKTHjLMphHnsEDDBu2qaV18ONSuFSQ-Sfg_Mg9ndS_IMTvS9IipsA
diff --git a/bm_mcc_mosk/seed/do_deploy_child.sh b/bm_mcc_mosk/seed/do_deploy_child.sh
new file mode 100644
index 0000000..6229555
--- /dev/null
+++ b/bm_mcc_mosk/seed/do_deploy_child.sh
@@ -0,0 +1,13 @@
+set -e
+while [ ! -d /root/bm_mcc_mosk/child/cluster ] ; do echo 'no child templates!' ; sleep 4 ; done
+export KUBECONFIG=/root/kubeconfig
+/root/kaas-bootstrap/bin/kubectl apply -f /root/bm_mcc_mosk/child/cluster/project.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/ssh_pubkeys.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/cluster.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/subnets.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/l2_templates.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/metallbconfig.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/machines.yaml
+sleep 2 && /root/kaas-bootstrap/bin/kubectl -n mosk apply -f /root/bm_mcc_mosk/child/cluster/kaascephcluster.yaml
diff --git a/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh b/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
new file mode 100644
index 0000000..b14994c
--- /dev/null
+++ b/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
@@ -0,0 +1,77 @@
+set -e
+source /root/bm_mcc_mosk/seed/env_vars.sh
+
+pushd /root/
+  wget https://binary.mirantis.com/releases/get_container_cloud.sh
+  chmod 0755 get_container_cloud.sh
+  ./get_container_cloud.sh
+  cp /root/mirantis.lic kaas-bootstrap/
+popd
+
+/root/kaas-bootstrap/bootstrap.sh bootstrapv2
+
+
+export KUBECONFIG=/root/.kube/kind-config-clusterapi
+pushd /root/bm_mcc_mosk/kaas-mgmt
+ /root/kaas-bootstrap/bin/kubectl apply -f bootstrapregion.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f serviceuser.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f cluster.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f baremetalhostprofiles.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f ipam-objects.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f baremetalhosts.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f metallbconfig.yaml && sleep 2
+ /root/kaas-bootstrap/bin/kubectl apply -f machines.yaml && sleep 2
+popd
+
+retry=60
+interval=60
+
+counter=0
+while (( counter++ < retry )) ; do
+  sleep $interval
+  avail_count=$(/root/kaas-bootstrap/bin/kubectl get bmh -o go-template='{{- range .items -}} {{.status.provisioning.state}}{{"\n"}} {{- end -}}' | (grep -c available || /bin/true))
+  echo "loop=${counter}, available state=${avail_count}"
+  if (( avail_count == 3 )); then
+    break
+  fi
+done
+if (( counter > retry )) ; then
+  echo "Not all nodes are available after timeout"
+  exit 1
+fi
+
+counter=0
+while (( counter++ < retry )) ; do
+  sleep $interval
+  status=$(/root/kaas-bootstrap/bin/kubectl get bootstrapregions -o go-template='{{(index .items 0).status.ready}}{{"\n"}}')
+  echo "loop=${counter}, status=${status}"
+  if [[ "$status" == "true" ]]; then
+    break
+  fi
+done
+if (( counter > retry )) ; then
+  echo "BootstrapRegion timed out"
+  exit 1
+fi
+
+/root/kaas-bootstrap/container-cloud bootstrap approve all
+
+counter=0
+retry=90
+while (( counter++ < retry )) ; do
+  sleep $interval
+  status=$(/root/kaas-bootstrap/bin/kubectl get cluster kaas-mgmt  -o go-template='{{.status.providerStatus.ready}}{{"\n"}}')
+  echo "loop=${counter}, status=${status}"
+  if [[ "$status" == "true" ]]; then
+    break
+  fi
+done
+if (( counter > retry )) ; then
+  echo "MGMT Cluster deployment timed out"
+  exit 1
+fi
+cd
+/root/kaas-bootstrap/container-cloud get cluster-kubeconfig --cluster-name kaas-mgmt --kubeconfig-output /root/kubeconfig
+echo r00tme | /root/kaas-bootstrap/container-cloud bootstrap user add --username root --roles global-admin,management-admin,reader,writer,operator --kubeconfig kubeconfig --password-stdin
+/root/kaas-bootstrap/bin/kind delete cluster -n clusterapi
+unset KUBECONFIG
diff --git a/bm_mcc_mosk/seed/env_vars.sh b/bm_mcc_mosk/seed/env_vars.sh
new file mode 100644
index 0000000..6252ad9
--- /dev/null
+++ b/bm_mcc_mosk/seed/env_vars.sh
@@ -0,0 +1,7 @@
+export KAAS_BM_ENABLED="true"
+export KAAS_BM_PXE_IP="172.16.180.5"
+export KAAS_BM_PXE_MASK="23"
+export KAAS_BM_PXE_BRIDGE="br0"
+export CLUSTER_NAME=kaas-mgmt
+export OUT_DIR=/root/bm_mcc_mosk/kaas-mgmt/
+#export KAAS_BOOTSTRAP_INFINITE_TIMEOUT=true
\ No newline at end of file
diff --git a/bm_mcc_mosk/seed/get_child_kubeconfig.sh b/bm_mcc_mosk/seed/get_child_kubeconfig.sh
new file mode 100644
index 0000000..23565e7
--- /dev/null
+++ b/bm_mcc_mosk/seed/get_child_kubeconfig.sh
@@ -0,0 +1,2 @@
+/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get secrets mosk-kubeconfig -o jsonpath='{.data.admin\.conf}' | base64 -d | sed 's/:5443/:443/g' > /root/child.kubeconfig
+export KUBECONFIG=/root/child.kubeconfig
\ No newline at end of file
diff --git a/jobs/pipelines/mosk/deploy-bm.groovy b/jobs/pipelines/mosk/deploy-bm.groovy
index a2cae19..ed4f920 100644
--- a/jobs/pipelines/mosk/deploy-bm.groovy
+++ b/jobs/pipelines/mosk/deploy-bm.groovy
@@ -59,8 +59,8 @@
           stage("Bootstrap MCC+child"){
              sh "scp ${ssh_params} ${MAINTENANCE_TEAM_SSH_KEY} root@${seed_ext_ip}:/root/.ssh/id_rsa "
              sh "scp -r ${ssh_params} bm_mcc_mosk root@${seed_ext_ip}: "
-             sh "ssh ${ssh_params} root@${seed_ext_ip} bash do_deploy_mcc_mgmt.sh"
-             sh "ssh ${ssh_params} root@${seed_ext_ip} bash do_deploy_child.sh"
+             sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh"
+             sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/do_deploy_child.sh"
              // TODO: unhardcode cluster names
              check_child_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get cluster mosk -o jsonpath='{.status.providerStatus.ready}' 2>/dev/null || echo 'ssh error, ignoring'"
              check_kcc_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get kcc mosk-ceph -o jsonpath='{.status.shortClusterInfo.state}' 2>/dev/null || echo 'ssh error, ignoring'"
@@ -84,7 +84,7 @@
                } //while
           } //stage MCC
           stage("Prepare and deploy MOSK"){
-            sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/get_child_kubeconfig.sh'"
+            sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/bm_mcc_mosk/seed/get_child_kubeconfig.sh'"
             sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/bm_mcc_mosk/utils/tsl_gen.sh'"
             kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
             sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/osdpl-ssl-secrets.yaml"
diff --git a/jobs/templates/bm-mcc-mosk-base.yml b/jobs/templates/bm-mcc-mosk-base.yml
index 90b8980..e68aaab 100644
--- a/jobs/templates/bm-mcc-mosk-base.yml
+++ b/jobs/templates/bm-mcc-mosk-base.yml
@@ -1,7 +1,7 @@
 - job-template:
     project-type: pipeline
     description: '{job-description}'
-    concurrent: true
+    concurrent: false
     disabled: false
     name: bm-mcc-mosk-base
     parameters:
@@ -50,7 +50,7 @@
         name: EXTRA_VARS
         trim: 'false'
     - bool:
-        default: true
+        default: false
         description: Run tempest after deploy
         name: RUN_TESTS_AFTER_DEPLOY
     pipeline-scm: