Restore BM lab after 17-3-0 release
- change disks mapping due to changed udev rules
- add grub parameter to future SRIOV support
- change DNS server to dnsmasq located at seed node
- add loadbalancer to subnets
- add cz7756 back to server pool
- remove coredns deployment
MOSSUST-122
Change-Id: Ifbd51516eb57955e5135f904e4d0581e74ee6dd6
diff --git a/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml b/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
index bd1fca9..b294c45 100644
--- a/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
+++ b/bm_mcc_mosk/child/cluster/baremetalhostprofiles.yaml
@@ -52,7 +52,7 @@
- 'GRUB_DISABLE_RECOVERY="true"'
- 'GRUB_PRELOAD_MODULES=lvm'
- 'GRUB_TIMEOUT=20'
- - 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
+ - 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0 "'
kernelParameters:
sysctl:
kernel.dmesg_restrict: "1"
@@ -81,8 +81,7 @@
spec:
devices:
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-1
+ hctl: "4:0:0:0"
minSize: 60Gi
wipeDevice:
eraseMetadata:
@@ -99,8 +98,7 @@
- name: root
size: 0
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-2
+ hctl: "5:0:0:0"
wipe: true
fileSystems:
- fileSystem: vfat
@@ -127,7 +125,7 @@
- 'GRUB_DISABLE_RECOVERY="true"'
- 'GRUB_PRELOAD_MODULES=lvm'
- 'GRUB_TIMEOUT=20'
- - 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
+ - 'GRUB_CMDLINE_LINUX="intel_iommu=on noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
kernelParameters:
sysctl:
kernel.dmesg_restrict: "1"
@@ -156,8 +154,7 @@
spec:
devices:
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-1
+ hctl: "4:0:0:0"
minSize: 60Gi
wipeDevice:
eraseMetadata:
@@ -174,8 +171,7 @@
- name: root
size: 0
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-2
+ hctl: "5:0:0:0"
wipe: true
fileSystems:
- fileSystem: vfat
@@ -202,7 +198,7 @@
- 'GRUB_DISABLE_RECOVERY="true"'
- 'GRUB_PRELOAD_MODULES=lvm'
- 'GRUB_TIMEOUT=20'
- - 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
+ - 'GRUB_CMDLINE_LINUX="intel_iommu=on noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
kernelParameters:
sysctl:
kernel.dmesg_restrict: "1"
@@ -227,8 +223,7 @@
spec:
devices:
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-1
+ hctl: "4:0:0:0"
minSize: 60Gi
wipeDevice:
eraseMetadata:
@@ -245,8 +240,7 @@
- name: root
size: 0
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-2
+ hctl: "5:0:0:0"
wipe: true
fileSystems:
- fileSystem: vfat
@@ -273,7 +267,7 @@
- 'GRUB_DISABLE_RECOVERY="true"'
- 'GRUB_PRELOAD_MODULES=lvm'
- 'GRUB_TIMEOUT=20'
- - 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
+ - 'GRUB_CMDLINE_LINUX="intel_iommu=on noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
kernelParameters:
sysctl:
kernel.dmesg_restrict: "1"
@@ -298,8 +292,7 @@
spec:
devices:
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-1
+ hctl: "4:0:0:0"
minSize: 60Gi
wipeDevice:
eraseMetadata:
@@ -316,8 +309,7 @@
- name: root
size: 0
- device:
- workBy: "by_id,by_path,by_wwn,by_name"
- byPath: /dev/disk/by-path/pci-0000:00:1f.2-ata-2
+ hctl: "5:0:0:0"
wipe: true
fileSystems:
- fileSystem: vfat
@@ -344,7 +336,7 @@
- 'GRUB_DISABLE_RECOVERY="true"'
- 'GRUB_PRELOAD_MODULES=lvm'
- 'GRUB_TIMEOUT=20'
- - 'GRUB_CMDLINE_LINUX="noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
+ - 'GRUB_CMDLINE_LINUX="intel_iommu=on noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off systemd.journald.forward_to_console=yes console=ttyS0,9600 console=tty0"'
kernelParameters:
sysctl:
kernel.dmesg_restrict: "1"
diff --git a/bm_mcc_mosk/child/cluster/baremetalhosts.yaml b/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
index df63c79..61ee873 100644
--- a/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
+++ b/bm_mcc_mosk/child/cluster/baremetalhosts.yaml
@@ -23,18 +23,18 @@
password:
value: KBwdcRqwed3w2
---
-#apiVersion: kaas.mirantis.com/v1alpha1
-#kind: BareMetalHostCredential
-#metadata:
-# name: cz7756-bmc-credentials
-# namespace: mosk
-# labels:
-# kaas.mirantis.com/provider: baremetal
-#spec:
-# username: engineer
-# password:
-# value: KBwdcRqwed3w2
-#---
+apiVersion: kaas.mirantis.com/v1alpha1
+kind: BareMetalHostCredential
+metadata:
+ name: cz7756-bmc-credentials
+ namespace: mosk
+ labels:
+ kaas.mirantis.com/provider: baremetal
+spec:
+ username: engineer
+ password:
+ value: KBwdcRqwed3w2
+---
apiVersion: kaas.mirantis.com/v1alpha1
kind: BareMetalHostCredential
metadata:
@@ -183,25 +183,25 @@
address: 185.8.59.229
credentialsName: 'cz7627-bmc-credentials'
---
-#apiVersion: metal3.io/v1alpha1
-#kind: BareMetalHost
-#metadata:
-# name: cz7756
-# labels:
-# kaas.mirantis.com/provider: baremetal
-# baremetal: hw-cz7756
-# "hostlabel.bm.kaas.mirantis.com/storage": "storage"
-# "hostlabel.bm.kaas.mirantis.com/worker": "worker"
-# "hostlabel.bm.kaas.mirantis.com/os_ctl": "os_ctl"
-# annotations:
-# "kaas.mirantis.com/baremetalhost-credentials-name": "cz7756-bmc-credentials"
-#spec:
-# bootMode: legacy
-# online: true
-# bootMACAddress: 0c:c4:7a:69:a0:4c
-# bmc:
-# address: 5.43.225.88
-# credentialsName: 'cz7756-bmc-credentials'
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+ name: cz7756
+ labels:
+ kaas.mirantis.com/provider: baremetal
+ baremetal: hw-cz7756
+ "hostlabel.bm.kaas.mirantis.com/storage": "storage"
+ "hostlabel.bm.kaas.mirantis.com/worker": "worker"
+ "hostlabel.bm.kaas.mirantis.com/os_ctl": "os_ctl"
+ annotations:
+ "kaas.mirantis.com/baremetalhost-credentials-name": "cz7756-bmc-credentials"
+spec:
+ bootMode: legacy
+ online: true
+ bootMACAddress: 0c:c4:7a:69:a0:4c
+ bmc:
+ address: 5.43.225.88
+ credentialsName: 'cz7756-bmc-credentials'
---
apiVersion: metal3.io/v1alpha1
kind: BareMetalHost
diff --git a/bm_mcc_mosk/child/cluster/cluster.yaml b/bm_mcc_mosk/child/cluster/cluster.yaml
index ec27963..133b388 100644
--- a/bm_mcc_mosk/child/cluster/cluster.yaml
+++ b/bm_mcc_mosk/child/cluster/cluster.yaml
@@ -16,10 +16,8 @@
providerSpec:
value:
kind: BaremetalClusterProviderSpec
- #release: mosk-17-2-4-24-2-2
- release: mosk-17-2-3-24-2-1
- loadBalancerHost: 172.16.180.250
-# loadBalancerHost: ""
+ release: mosk-17-3-0-24-3
+ loadBalancerHost: ''
apiVersion: baremetal.k8s.io/v1alpha1
dedicatedControlPlane: false
dedicatedMetallbPools: false
@@ -31,8 +29,6 @@
- name: pglazov
- name: dmeltsaykin
- name: rlubianyi
- dnsNameservers:
- - 172.18.176.6
helmReleases:
- enabled: true
name: ceph-controller
@@ -51,7 +47,4 @@
kaas:
management:
enabled: false
-# provider: baremetal
keycloak: {}
- release: kaas-2-26-0
-# nodeCidr: 10.10.10.0/24
diff --git a/bm_mcc_mosk/child/cluster/subnets.yaml b/bm_mcc_mosk/child/cluster/subnets.yaml
index cd8bb1e..f0b57e2 100644
--- a/bm_mcc_mosk/child/cluster/subnets.yaml
+++ b/bm_mcc_mosk/child/cluster/subnets.yaml
@@ -14,6 +14,20 @@
nameservers:
- 172.18.176.6
---
+apiVersion: "ipam.mirantis.com/v1alpha1"
+kind: Subnet
+# ONE ip addr! for cluster:spec:loadBalancerHost
+metadata:
+ name: k8s-api-lb
+ namespace: mosk
+ labels:
+ kaas.mirantis.com/provider: baremetal
+ cluster.sigs.k8s.io/cluster-name: mosk
+ ipam/SVC-LBhost: "1"
+spec:
+ cidr: 172.16.180.250/32
+ useWholeCidr: true
+---
apiVersion: ipam.mirantis.com/v1alpha1
kind: Subnet
metadata:
diff --git a/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh b/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
index 960427a..56415bf 100755
--- a/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
+++ b/bm_mcc_mosk/seed/do_deploy_mcc_mgmt.sh
@@ -84,4 +84,4 @@
/root/kaas-bootstrap/bin/kind delete cluster -n clusterapi
unset KUBECONFIG
-bash /root/bm_mcc_mosk/utils/update_bash_autocompletion_kaas.sh
\ No newline at end of file
+bash /root/bm_mcc_mosk/utils/update_bash_autocompletion_kaas.sh
diff --git a/bm_mcc_mosk/seed/setup_dnsmasq.sh b/bm_mcc_mosk/seed/setup_dnsmasq.sh
index d9f110f..ac0c4b5 100755
--- a/bm_mcc_mosk/seed/setup_dnsmasq.sh
+++ b/bm_mcc_mosk/seed/setup_dnsmasq.sh
@@ -19,7 +19,6 @@
EOF
LISTEN_ADDR=$(ip a | grep 172 | awk '{print $2}' | awk -F/ '{print $1}' | head -n1)
grep "${LISTEN_ADDR}" /etc/dnsmasq.conf || echo "listen-address=::1,127.0.0.1,${LISTEN_ADDR}" >> /etc/dnsmasq.conf
-grep "subdomain.team.sustaining" /etc/dnsmasq.conf || echo "server=/*.subdomain.team.sustaining/${1}" >> /etc/dnsmasq.conf
-grep "mnt-bm.mirantis.net" /etc/dnsmasq.conf || echo "server=/*.mnt-bm.mirantis.net/${1}" >> /etc/dnsmasq.conf
+grep "mnt-bm.mirantis.net" /etc/dnsmasq.conf || echo "address=/.mnt-bm.mirantis.net/${1}" >> /etc/dnsmasq.conf
grep "server=172.18.176.6" /etc/dnsmasq.conf || echo "server=172.18.176.6" >> /etc/dnsmasq.conf
service dnsmasq restart
diff --git a/bm_mcc_mosk/utils/update_bash_autocompletion_kaas.sh b/bm_mcc_mosk/utils/update_bash_autocompletion_kaas.sh
index 1e5e23e..6be1dac 100755
--- a/bm_mcc_mosk/utils/update_bash_autocompletion_kaas.sh
+++ b/bm_mcc_mosk/utils/update_bash_autocompletion_kaas.sh
@@ -9,20 +9,20 @@
if [ -f ~/child.kubeconfig ]; then export KUBECONFIG=~/child.kubeconfig; fi
if [[ -f ~/clouds.yaml && -f /usr/local/bin/yq ]] ; then
export OS_CLOUD=admin
- OS_USERNAME=$(yq .clouds.admin.auth.username clouds.yaml)
- OS_PASSWORD=$(yq .clouds.admin.auth.password clouds.yaml)
- OS_AUTH_URL=$(yq .clouds.admin.auth.auth_url clouds.yaml)
+ OS_USERNAME=\$(yq .clouds.admin.auth.username clouds.yaml)
+ OS_PASSWORD=\$(yq .clouds.admin.auth.password clouds.yaml)
+ OS_AUTH_URL=\$(yq .clouds.admin.auth.auth_url clouds.yaml)
fi
echo
-echo "KUBECONFIG=${KUBECONFIG}"
-echo "kubectl=$(which kubectl)"
+echo "KUBECONFIG=\${KUBECONFIG}"
+echo "kubectl=\$(which kubectl)"
echo
-echo "OS_USERNAME=${OS_USERNAME}"
-echo "OS_PASSWORD=${OS_PASSWORD}"
+echo "OS_USERNAME=\${OS_USERNAME}"
+echo "OS_PASSWORD=\${OS_PASSWORD}"
echo
[[ -f ~/openstack_clients/bin/activate ]] && echo "OpenStack shell clients at 'source ~/openstack_clients/bin/activate'"
echo
-echo "Horizon at ${OS_AUTH_URL/keystone/horizon}"
+echo "Horizon at \${OS_AUTH_URL/keystone/horizon}"
echo
EOF
diff --git a/jobs/pipelines/mosk/deploy-bm.groovy b/jobs/pipelines/mosk/deploy-bm.groovy
index 4b3c23f..1967a99 100644
--- a/jobs/pipelines/mosk/deploy-bm.groovy
+++ b/jobs/pipelines/mosk/deploy-bm.groovy
@@ -101,18 +101,11 @@
} //stage MOSK
stage("Configure DNS server") {
get_ingress_ip_cmd = "${kubectl_openstack_cmd} get service ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}'"
- get_dns_internal_ip_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n coredns get service coredns-coredns -o jsonpath='{.spec.clusterIP}'"
- get_dns_external_ip_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n coredns get service coredns-coredns -o jsonpath='{.status.loadBalancer.ingress[0].ip}'"
ingress_ip = sh(returnStdout: true, script: get_ingress_ip_cmd).trim()
- sh "ssh ${ssh_params} root@${seed_ext_ip} sed -i 's/!!!EXT_DNS_IP/${ingress_ip}/g' /root/bm_mcc_mosk/child/kaas_workloads/coredns.yaml"
- sh "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig apply -f /root/bm_mcc_mosk/child/kaas_workloads/coredns.yaml"
- sleep 120
- dns_internal_ip = sh(returnStdout: true, script: get_dns_internal_ip_cmd).trim()
+ sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/setup_dnsmasq.sh ${ingress_ip}"
sh "ssh ${ssh_params} root@${seed_ext_ip} \"/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n kube-system get configmap coredns -oyaml > coredns.conf\""
- sh "ssh ${ssh_params} root@${seed_ext_ip} /usr/bin/python3 /root/bm_mcc_mosk/utils/update_kube_dns_conf.py ${dns_internal_ip}"
+ sh "ssh ${ssh_params} root@${seed_ext_ip} /usr/bin/python3 /root/bm_mcc_mosk/utils/update_kube_dns_conf.py ${seed_ext_ip}"
sh "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig apply -f coredns.patched.conf"
- dns_external_ip = sh(returnStdout: true, script: get_dns_external_ip_cmd).trim()
- sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/setup_dnsmasq.sh ${dns_external_ip}"
sh "ssh ${ssh_params} root@${seed_ext_ip} bash /root/bm_mcc_mosk/seed/get_openstack_credentials.sh"
} // stage DNS