Revert "Revert "Add k8s-ha-calico model support""
This reverts commit 3b1f40e9da7cd4a7a91938f7fe2c66c01cf4d494.
Change-Id: I5871f91c3fa9d6cfa65afb1d4914eea0acabe35c
diff --git a/tcp_tests/templates/k8s-ha-calico/common-services.yaml b/tcp_tests/templates/k8s-ha-calico/common-services.yaml
new file mode 100644
index 0000000..5c0fcf9
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/common-services.yaml
@@ -0,0 +1,58 @@
+{% from 'k8s-ha-calico/underlay.yaml' import HOSTNAME_CFG01 with context %}
+
+# Install support services
+- description: Create and distribute SSL certificates for services using salt state
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*'
+ state.sls salt.minion.cert
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: true
+
+- description: Install docker
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@docker:host' state.sls docker.host
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 3, delay: 5}
+ skip_fail: false
+
+- description: Check docker
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@docker:host' cmd.run 'docker ps'
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Install keepalived on first node
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@keepalived:cluster and *01*' state.sls keepalived
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: true
+
+- description: Install keepalived on whole cluster
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@keepalived:cluster and *01*' state.sls keepalived
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 10}
+ skip_fail: true
+
+- description: Install haproxy
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@haproxy:proxy' state.sls haproxy
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Install haproxy.service
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@haproxy:proxy' service.status haproxy
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Restart rsyslog on haproxy nodes
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@haproxy:proxy' service.restart rsyslog
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
diff --git a/tcp_tests/templates/k8s-ha-calico/k8s.yaml b/tcp_tests/templates/k8s-ha-calico/k8s.yaml
new file mode 100644
index 0000000..e443835
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/k8s.yaml
@@ -0,0 +1,79 @@
+{% from 'k8s-ha-calico/underlay.yaml' import HOSTNAME_CFG01 with context %}
+
+# Kubernetes
+- description: Install etcd
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@etcd:server' state.sls etcd.server.service
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 3, delay: 5}
+ skip_fail: false
+
+- description: Check the etcd health
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@etcd:server' cmd.run '. /var/lib/etcd/configenv && etcdctl cluster-health'
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Install Kubernetes addons
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@kubernetes:master' state.sls kubernetes.master.kube-addons
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 3, delay: 5}
+ skip_fail: false
+
+- description: Install Kubernetes and Calico
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@kubernetes:pool' state.sls kubernetes.pool
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 3, delay: 5}
+ skip_fail: false
+
+- description: Setup NAT for Calico
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@etcd:server and *01*' state.sls etcd.server.setup
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Run whole master to check consistency
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@kubernetes:master' state.sls kubernetes exclude=kubernetes.master.setup
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: true
+
+- description: Register addons
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@kubernetes:master and *01*' state.sls kubernetes.master.setup
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Restart kubelet
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@kubernetes:pool' service.restart kubelet
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Renew hosts file on a whole cluster
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '*' state.sls linux.network.host;
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Get kubeconfig
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@kubernetes:pool and *01*' cmd.run 'cat /etc/kubernetes/admin-kube-config && echo "Salt command execution success"'
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Install tiller client
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@helm:client' test.ping && salt --hard-crash --state-output=mixed --state-verbose=False
+ -C 'I@helm:client' state.sls helm
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 3, delay: 15}
+ skip_fail: true
diff --git a/tcp_tests/templates/k8s-ha-calico/salt.yaml b/tcp_tests/templates/k8s-ha-calico/salt.yaml
new file mode 100644
index 0000000..2ad0498
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/salt.yaml
@@ -0,0 +1,26 @@
+{% from 'k8s-ha-calico/underlay.yaml' import HOSTNAME_CFG01 with context %}
+{% from 'k8s-ha-calico/underlay.yaml' import LAB_CONFIG_NAME with context %}
+{% from 'k8s-ha-calico/underlay.yaml' import DOMAIN_NAME with context %}
+
+{% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
+# Other salt model repository parameters see in shared-salt.yaml
+
+{% import 'shared-salt.yaml' as SHARED with context %}
+
+{{ SHARED.MACRO_INSTALL_SALT_MASTER() }}
+
+{{ SHARED.MACRO_CLONE_RECLASS_MODELS() }}
+
+{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch"') }}
+
+{{ SHARED.MACRO_INSTALL_SALT_MINIONS() }}
+
+{{ SHARED.MACRO_RUN_SALT_MASTER_UNDERLAY_STATES() }}
+
+{{ SHARED.ADJUST_K8S_OPTS() }}
+
+{{ SHARED.REGISTER_COMPUTE_NODES() }}
+
+{{ SHARED.MACRO_GENERATE_INVENTORY() }}
+
+{{ SHARED.MACRO_BOOTSTRAP_ALL_MINIONS() }}
diff --git a/tcp_tests/templates/k8s-ha-calico/underlay--meta-data.yaml b/tcp_tests/templates/k8s-ha-calico/underlay--meta-data.yaml
new file mode 100644
index 0000000..3699401
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/underlay--meta-data.yaml
@@ -0,0 +1,4 @@
+| # All the data below will be stored as a string object
+ instance-id: iid-local1
+ hostname: {hostname}
+ local-hostname: {hostname}
diff --git a/tcp_tests/templates/k8s-ha-calico/underlay--user-data-cfg01.yaml b/tcp_tests/templates/k8s-ha-calico/underlay--user-data-cfg01.yaml
new file mode 100644
index 0000000..e37eaa0
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/underlay--user-data-cfg01.yaml
@@ -0,0 +1,86 @@
+| # All the data below will be stored as a string object
+ #cloud-config, see http://cloudinit.readthedocs.io/en/latest/topics/examples.html
+
+ ssh_pwauth: True
+ users:
+ - name: root
+ sudo: ALL=(ALL) NOPASSWD:ALL
+ shell: /bin/bash
+ ssh_authorized_keys:
+ {% for key in config.underlay.ssh_keys %}
+ - ssh-rsa {{ key['public'] }}
+ {% endfor %}
+
+ disable_root: false
+ chpasswd:
+ list: |
+ root:r00tme
+ expire: False
+
+ bootcmd:
+ # Block access to SSH while node is preparing
+ - cloud-init-per once sudo iptables -A INPUT -p tcp --dport 22 -j DROP
+ # Enable root access
+ - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin yes/' /etc/ssh/sshd_config
+ - service sshd restart
+ output:
+ all: '| tee -a /var/log/cloud-init-output.log /dev/tty0'
+
+ runcmd:
+ # Configure dhclient
+ - sudo echo "nameserver {gateway}" >> /etc/resolvconf/resolv.conf.d/base
+ - sudo resolvconf -u
+
+ # Prepare network connection
+ - sudo ifup ens3
+ #- sudo route add default gw {gateway} {interface_name}
+ - sudo ifup ens4
+
+ # Create swap
+ - fallocate -l 4G /swapfile
+ - chmod 600 /swapfile
+ - mkswap /swapfile
+ - swapon /swapfile
+ - echo "/swapfile none swap defaults 0 0" >> /etc/fstab
+
+ ############## TCP Cloud cfg01 node ##################
+ #- sleep 120
+ - echo "Preparing base OS"
+
+ - echo "nameserver 172.18.208.44" >> /etc/resolv.conf;
+ - echo "nameserver 8.8.8.8" >> /etc/resolv.conf;
+ - which wget >/dev/null || (apt-get update; apt-get install -y wget);
+
+ - echo "deb [arch=amd64] http://apt.mirantis.com/xenial {{ REPOSITORY_SUITE }} salt extra" > /etc/apt/sources.list.d/mcp_salt.list;
+ - wget -O - http://apt.mirantis.com/public.gpg | apt-key add -;
+ - echo "deb http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3 xenial main" > /etc/apt/sources.list.d/saltstack.list;
+ - wget -O - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3/SALTSTACK-GPG-KEY.pub | apt-key add -;
+
+ - eatmydata apt-get clean && apt-get update
+
+ # Install common packages
+ - eatmydata apt-get install -y python-pip git curl tmux byobu iputils-ping traceroute htop tree
+
+ ########################################################
+ # Node is ready, allow SSH access
+ - echo "Allow SSH access ..."
+ - sudo iptables -D INPUT -p tcp --dport 22 -j DROP
+ ########################################################
+
+ write_files:
+ - path: /etc/network/interfaces
+ content: |
+ auto ens3
+ iface ens3 inet dhcp
+ auto ens4
+ iface ens4 inet dhcp
+
+ - path: /root/.ssh/config
+ owner: root:root
+ permissions: '0600'
+ content: |
+ Host *
+ ServerAliveInterval 300
+ ServerAliveCountMax 10
+ StrictHostKeyChecking no
+ UserKnownHostsFile /dev/null
diff --git a/tcp_tests/templates/k8s-ha-calico/underlay--user-data1604.yaml b/tcp_tests/templates/k8s-ha-calico/underlay--user-data1604.yaml
new file mode 100644
index 0000000..97be4b0
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/underlay--user-data1604.yaml
@@ -0,0 +1,77 @@
+| # All the data below will be stored as a string object
+ #cloud-config, see http://cloudinit.readthedocs.io/en/latest/topics/examples.html
+
+ ssh_pwauth: True
+ users:
+ - name: root
+ sudo: ALL=(ALL) NOPASSWD:ALL
+ shell: /bin/bash
+ ssh_authorized_keys:
+ {% for key in config.underlay.ssh_keys %}
+ - ssh-rsa {{ key['public'] }}
+ {% endfor %}
+
+ disable_root: false
+ chpasswd:
+ list: |
+ root:r00tme
+ expire: False
+
+ bootcmd:
+ # Block access to SSH while node is preparing
+ - cloud-init-per once sudo iptables -A INPUT -p tcp --dport 22 -j DROP
+ # Enable root access
+ - sed -i -e '/^PermitRootLogin/s/^.*$/PermitRootLogin yes/' /etc/ssh/sshd_config
+ - service sshd restart
+ output:
+ all: '| tee -a /var/log/cloud-init-output.log /dev/tty0'
+
+ runcmd:
+ - export TERM=linux
+ - export LANG=C
+ # Configure dhclient
+ - sudo echo "nameserver {gateway}" >> /etc/resolvconf/resolv.conf.d/base
+ - sudo resolvconf -u
+
+ # Prepare network connection
+ - sudo ifup ens3
+ #- sudo route add default gw {gateway} {interface_name}
+ - sudo ifup ens4
+
+ # Create swap
+ - fallocate -l 4G /swapfile
+ - chmod 600 /swapfile
+ - mkswap /swapfile
+ - swapon /swapfile
+ - echo "/swapfile none swap defaults 0 0" >> /etc/fstab
+
+
+ ############## TCP Cloud cfg01 node ##################
+ #- sleep 120
+ - echo "Preparing base OS"
+ - which wget >/dev/null || (apt-get update; apt-get install -y wget)
+
+ - echo "deb [arch=amd64] http://apt.mirantis.com/xenial {{ REPOSITORY_SUITE }} salt extra" > /etc/apt/sources.list.d/mcp_salt.list;
+ - wget -O - http://apt.mirantis.com/public.gpg | apt-key add -;
+ - echo "deb http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3 xenial main" > /etc/apt/sources.list.d/saltstack.list;
+ - wget -O - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.3/SALTSTACK-GPG-KEY.pub | apt-key add -;
+
+ - apt-get clean
+ - eatmydata apt-get update && apt-get -y upgrade
+
+ # Install common packages
+ - eatmydata apt-get install -y python-pip git curl tmux byobu iputils-ping traceroute htop tree mc
+
+ ########################################################
+ # Node is ready, allow SSH access
+ - echo "Allow SSH access ..."
+ - sudo iptables -D INPUT -p tcp --dport 22 -j DROP
+ ########################################################
+
+ write_files:
+ - path: /etc/network/interfaces
+ content: |
+ auto ens3
+ iface ens3 inet dhcp
+ auto ens4
+ iface ens4 inet dhcp
diff --git a/tcp_tests/templates/k8s-ha-calico/underlay.yaml b/tcp_tests/templates/k8s-ha-calico/underlay.yaml
new file mode 100644
index 0000000..ac1bd95
--- /dev/null
+++ b/tcp_tests/templates/k8s-ha-calico/underlay.yaml
@@ -0,0 +1,293 @@
+# Set the repository suite, one of the: 'nightly', 'testing', 'stable', or any other required
+{% set REPOSITORY_SUITE = os_env('REPOSITORY_SUITE', 'testing') %}
+
+{% import 'k8s-ha-calico/underlay--meta-data.yaml' as CLOUDINIT_META_DATA with context %}
+{% import 'k8s-ha-calico/underlay--user-data-cfg01.yaml' as CLOUDINIT_USER_DATA_CFG01 with context %}
+{% import 'k8s-ha-calico/underlay--user-data1604.yaml' as CLOUDINIT_USER_DATA_1604 with context %}
+
+---
+aliases:
+ - &interface_model {{ os_env('INTERFACE_MODEL', 'virtio') }}
+ - &cloudinit_meta_data {{ CLOUDINIT_META_DATA }}
+ - &cloudinit_user_data_cfg01 {{ CLOUDINIT_USER_DATA_CFG01 }}
+ - &cloudinit_user_data_1604 {{ CLOUDINIT_USER_DATA_1604 }}
+
+{% set LAB_CONFIG_NAME = os_env('LAB_CONFIG_NAME', 'k8s-ha-calico.local') %}
+{% set DOMAIN_NAME = os_env('DOMAIN_NAME', LAB_CONFIG_NAME) + '.local' %}
+{% set HOSTNAME_CFG01 = os_env('HOSTNAME_CFG01', 'cfg01.' + DOMAIN_NAME) %}
+{% set HOSTNAME_CTL01 = os_env('HOSTNAME_CTL01', 'ctl01.' + DOMAIN_NAME) %}
+{% set HOSTNAME_CTL02 = os_env('HOSTNAME_CTL02', 'ctl02.' + DOMAIN_NAME) %}
+{% set HOSTNAME_CTL03 = os_env('HOSTNAME_CTL03', 'ctl03.' + DOMAIN_NAME) %}
+{% set HOSTNAME_CMP01 = os_env('HOSTNAME_CMP01', 'cmp0.' + DOMAIN_NAME) %}
+{% set HOSTNAME_CMP02 = os_env('HOSTNAME_CMP02', 'cmp1.' + DOMAIN_NAME) %}
+
+template:
+ devops_settings:
+ env_name: {{ os_env('ENV_NAME', 'k8s-ha-calico_' + REPOSITORY_SUITE + "_" + os_env('BUILD_NUMBER', '')) }}
+
+ address_pools:
+ admin-pool01:
+ net: {{ os_env('ADMIN_ADDRESS_POOL01', '10.70.0.0/16:24') }}
+ params:
+ ip_reserved:
+ gateway: +1
+ l2_network_device: +1
+ default_{{ HOSTNAME_CFG01 }}: +90
+ default_{{ HOSTNAME_CTL01 }}: +101
+ default_{{ HOSTNAME_CTL02 }}: +102
+ default_{{ HOSTNAME_CTL03 }}: +103
+ default_{{ HOSTNAME_CMP01 }}: +105
+ default_{{ HOSTNAME_CMP02 }}: +106
+ ip_ranges:
+ dhcp: [+90, -10]
+
+ private-pool01:
+ net: {{ os_env('PRIVATE_ADDRESS_POOL01', '10.60.0.0/16:24') }}
+ params:
+ ip_reserved:
+ gateway: +1
+ l2_network_device: +1
+ default_{{ HOSTNAME_CFG01 }}: +100
+ default_{{ HOSTNAME_CTL01 }}: +101
+ default_{{ HOSTNAME_CTL02 }}: +102
+ default_{{ HOSTNAME_CTL03 }}: +103
+ default_{{ HOSTNAME_CMP01 }}: +105
+ default_{{ HOSTNAME_CMP02 }}: +106
+ ip_ranges:
+ dhcp: [+90, -10]
+
+ groups:
+ - name: default
+ driver:
+ name: devops.driver.libvirt
+ params:
+ connection_string: !os_env CONNECTION_STRING, qemu:///system
+ storage_pool_name: !os_env STORAGE_POOL_NAME, default
+ stp: False
+ hpet: False
+ enable_acpi: true
+ use_host_cpu: !os_env DRIVER_USE_HOST_CPU, true
+ use_hugepages: !os_env DRIVER_USE_HUGEPAGES, false
+
+ network_pools:
+ admin: admin-pool01
+ private: private-pool01
+
+ l2_network_devices:
+ private:
+ address_pool: private-pool01
+ dhcp: true
+
+ admin:
+ address_pool: admin-pool01
+ dhcp: true
+ forward:
+ mode: nat
+
+ group_volumes:
+ - name: cloudimage1604 # This name is used for 'backing_store' option for node volumes.
+ source_image: !os_env IMAGE_PATH1604 # https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img or
+ # http://apt.tcpcloud.eu/images/ubuntu-16-04-x64-201608231004.qcow2
+ format: qcow2
+
+ nodes:
+ - name: {{ HOSTNAME_CFG01 }}
+ role: salt_master
+ params:
+ vcpu: !os_env SLAVE_NODE_CPU, 2
+ memory: !os_env SLAVE_NODE_MEMORY, 4096
+ boot:
+ - hd
+ cloud_init_volume_name: iso
+ cloud_init_iface_up: ens3
+ volumes:
+ - name: system
+ capacity: !os_env NODE_VOLUME_SIZE, 150
+ backing_store: cloudimage1604
+ format: qcow2
+ - name: iso # Volume with name 'iso' will be used
+ # for store image with cloud-init metadata.
+ capacity: 1
+ format: raw
+ device: cdrom
+ bus: ide
+ cloudinit_meta_data: *cloudinit_meta_data
+ cloudinit_user_data: *cloudinit_user_data_cfg01
+
+ interfaces:
+ - label: ens3
+ l2_network_device: admin
+ interface_model: *interface_model
+ - label: ens4
+ l2_network_device: private
+ interface_model: *interface_model
+ network_config:
+ ens3:
+ networks:
+ - admin
+ ens4:
+ networks:
+ - private
+
+ - name: {{ HOSTNAME_CTL01 }}
+ role: k8s_controller
+ params:
+ vcpu: !os_env SLAVE_NODE_CPU, 2
+ memory: !os_env SLAVE_NODE_MEMORY, 4096
+ boot:
+ - hd
+ cloud_init_volume_name: iso
+ cloud_init_iface_up: ens3
+ volumes:
+ - name: system
+ capacity: !os_env NODE_VOLUME_SIZE, 150
+ backing_store: cloudimage1604
+ format: qcow2
+ - name: cinder
+ capacity: 50
+ format: qcow2
+ - name: iso # Volume with name 'iso' will be used
+ # for store image with cloud-init metadata.
+ capacity: 1
+ format: raw
+ device: cdrom
+ bus: ide
+ cloudinit_meta_data: *cloudinit_meta_data
+ cloudinit_user_data: *cloudinit_user_data_1604
+
+ interfaces: &interfaces
+ - label: ens3
+ l2_network_device: admin
+ interface_model: *interface_model
+ - label: ens4
+ l2_network_device: private
+ interface_model: *interface_model
+ network_config: &network_config
+ ens3:
+ networks:
+ - admin
+ ens4:
+ networks:
+ - private
+
+ - name: {{ HOSTNAME_CTL02 }}
+ role: salt_minion
+ params:
+ vcpu: !os_env SLAVE_NODE_CPU, 2
+ memory: !os_env SLAVE_NODE_MEMORY, 4096
+ boot:
+ - hd
+ cloud_init_volume_name: iso
+ cloud_init_iface_up: ens3
+ volumes:
+ - name: system
+ capacity: !os_env NODE_VOLUME_SIZE, 150
+ backing_store: cloudimage1604
+ format: qcow2
+ - name: cinder
+ capacity: 50
+ format: qcow2
+ - name: iso # Volume with name 'iso' will be used
+ # for store image with cloud-init metadata.
+ capacity: 1
+ format: raw
+ device: cdrom
+ bus: ide
+ cloudinit_meta_data: *cloudinit_meta_data
+ cloudinit_user_data: *cloudinit_user_data_1604
+
+ interfaces: *interfaces
+ network_config: *network_config
+
+ - name: {{ HOSTNAME_CTL03 }}
+ role: salt_minion
+ params:
+ vcpu: !os_env SLAVE_NODE_CPU, 2
+ memory: !os_env SLAVE_NODE_MEMORY, 4096
+ boot:
+ - hd
+ cloud_init_volume_name: iso
+ cloud_init_iface_up: ens3
+ volumes:
+ - name: system
+ capacity: !os_env NODE_VOLUME_SIZE, 150
+ backing_store: cloudimage1604
+ format: qcow2
+ - name: cinder
+ capacity: 50
+ format: qcow2
+ - name: iso # Volume with name 'iso' will be used
+ # for store image with cloud-init metadata.
+ capacity: 1
+ format: raw
+ device: cdrom
+ bus: ide
+ cloudinit_meta_data: *cloudinit_meta_data
+ cloudinit_user_data: *cloudinit_user_data_1604
+
+ interfaces: *interfaces
+ network_config: *network_config
+
+ - name: {{ HOSTNAME_CMP01 }}
+ role: salt_minion
+ params:
+ vcpu: !os_env SLAVE_NODE_CPU, 3
+ memory: !os_env SLAVE_NODE_MEMORY, 4096
+ boot:
+ - hd
+ cloud_init_volume_name: iso
+ cloud_init_iface_up: ens3
+ volumes:
+ - name: system
+ capacity: !os_env NODE_VOLUME_SIZE, 150
+ backing_store: cloudimage1604
+ format: qcow2
+ - name: iso # Volume with name 'iso' will be used
+ # for store image with cloud-init metadata.
+ capacity: 1
+ format: raw
+ device: cdrom
+ bus: ide
+ cloudinit_meta_data: *cloudinit_meta_data
+ cloudinit_user_data: *cloudinit_user_data_1604
+
+ interfaces: &all_interfaces
+ - label: ens3
+ l2_network_device: admin
+ interface_model: *interface_model
+ - label: ens4
+ l2_network_device: private
+ interface_model: *interface_model
+ network_config: &all_network_config
+ ens3:
+ networks:
+ - admin
+ ens4:
+ networks:
+ - private
+
+ - name: {{ HOSTNAME_CMP02 }}
+ role: salt_minion
+ params:
+ vcpu: !os_env SLAVE_NODE_CPU, 3
+ memory: !os_env SLAVE_NODE_MEMORY, 4096
+ boot:
+ - hd
+ cloud_init_volume_name: iso
+ cloud_init_iface_up: ens3
+ volumes:
+ - name: system
+ capacity: !os_env NODE_VOLUME_SIZE, 150
+ backing_store: cloudimage1604
+ format: qcow2
+ - name: iso # Volume with name 'iso' will be used
+ # for store image with cloud-init metadata.
+ capacity: 1
+ format: raw
+ device: cdrom
+ bus: ide
+ cloudinit_meta_data: *cloudinit_meta_data
+ cloudinit_user_data: *cloudinit_user_data_1604
+
+ interfaces: *interfaces
+ network_config: *network_config
diff --git a/tcp_tests/templates/shared-salt.yaml b/tcp_tests/templates/shared-salt.yaml
index efd894b..145c57f 100644
--- a/tcp_tests/templates/shared-salt.yaml
+++ b/tcp_tests/templates/shared-salt.yaml
@@ -594,15 +594,15 @@
skip_fail: false
- description: Check salt minion versions on slaves
- cmd: salt '*' test.version
+ cmd: salt --timeout=60 '*' test.version
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 3, delay: 15}
skip_fail: false
- description: Check salt top states on nodes
- cmd: salt '*' state.show_top
+ cmd: salt --timeout=60 '*' state.show_top
node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
+ retry: {count: 3, delay: 15}
skip_fail: false
- description: Configure ntp and rsyslog on nodes
@@ -657,3 +657,182 @@
skip_fail: false
{%- endmacro %}
+
+{%- macro REGISTER_COMPUTE_NODES() %}
+{#########################################}
+
+{% for ssh in config.underlay.ssh %}
+{% if ssh["node_name"].startswith("cmp") %}
+{% set node_hostname = ssh["node_name"].split('.')[0] %}
+- description: Register compute {{ ssh['node_name'] }}
+ cmd: |
+ exec > >(tee -i /tmp/cloud-init-bootstrap.log) 2>&1
+ export network01_prefix={{ IPV4_NET_ADMIN_PREFIX }}
+ export network02_prefix={{ IPV4_NET_CONTROL_PREFIX }}
+ export cluster_name={{ LAB_CONFIG_NAME }}
+ export config_host={{ config.salt.salt_master_host }}
+ export node_domain={{ DOMAIN_NAME }}
+ export nodes_os="xenial"
+ export node_hostname={{ node_hostname }}
+ set -xe
+ export BOOTSTRAP_SCRIPT_URL=$bootstrap_script_url
+ export BOOTSTRAP_SCRIPT_URL=${BOOTSTRAP_SCRIPT_URL:-https://raw.githubusercontent.com/salt-formulas/salt-formulas-scripts/master/bootstrap.sh}
+ export DISTRIB_REVISION={{ REPOSITORY_SUITE }}
+ export DISTRIB_REVISION=${DISTRIB_REVISION:-nightly}
+
+ # Add wrapper to apt-get to avoid race conditions
+ # with cron jobs running 'unattended-upgrades' script
+ aptget_wrapper() {
+ local apt_wrapper_timeout=300
+ local start_time=$(date '+%s')
+ local fin_time=$((start_time + apt_wrapper_timeout))
+ while true; do
+ if (( "$(date '+%s')" > fin_time )); then
+ msg="Timeout exceeded ${apt_wrapper_timeout} s. Lock files are still not released. Terminating..."
+ fi
+ if fuser /var/lib/apt/lists/lock >/dev/null 2>&1 || fuser /var/lib/dpkg/lock >/dev/null 2>&1; then
+ echo "Waiting while another apt/dpkg process releases locks ..."
+ sleep 30
+ continue
+ else
+ apt-get $@
+ break
+ fi
+ done
+ }
+
+ # Set default salt version
+ if [ -z "$saltversion" ]; then
+ saltversion="2016.3"
+ fi
+ echo "Using Salt version $saltversion"
+
+ echo "Preparing base OS ..."
+
+ case "$node_os" in
+ trusty)
+ # workaround for old cloud-init only configuring the first iface
+ iface_config_dir="/etc/network/interfaces"
+ ifaces=$(ip a | awk '/^[1-9]:/ {print $2}' | grep -v "lo:" | rev | cut -c2- | rev)
+
+ for iface in $ifaces; do
+ grep $iface $iface_config_dir &> /dev/null || (echo -e "\nauto $iface\niface $iface inet dhcp" >> $iface_config_dir && ifup $iface)
+ done
+
+ which wget > /dev/null || (aptget_wrapper update; aptget_wrapper install -y wget)
+
+ # SUGGESTED UPDATE:
+ #export MASTER_IP="$config_host" MINION_ID="$node_hostname.$node_domain" SALT_VERSION=$saltversion
+ #source <(curl -qL ${BOOTSTRAP_SCRIPT_URL})
+ ## Update BOOTSTRAP_SALTSTACK_OPTS, as by default they contain "-dX" not to start service
+ #BOOTSTRAP_SALTSTACK_OPTS=" stable $SALT_VERSION "
+ #install_salt_minion_pkg
+
+ # DEPRECATED:
+ echo "deb [arch=amd64] http://apt-mk.mirantis.com/trusty ${DISTRIB_REVISION} salt extra" > /etc/apt/sources.list.d/mcp_salt.list
+ wget -O - http://apt-mk.mirantis.com/public.gpg | apt-key add -
+
+ echo "deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/$saltversion trusty main" > /etc/apt/sources.list.d/saltstack.list
+ wget -O - "https://repo.saltstack.com/apt/ubuntu/14.04/amd64/$saltversion/SALTSTACK-GPG-KEY.pub" | apt-key add -
+
+ aptget_wrapper clean
+ aptget_wrapper update
+ aptget_wrapper install -y salt-common
+ aptget_wrapper install -y salt-minion
+ ;;
+ xenial)
+
+ # workaround for new cloud-init setting all interfaces statically
+ which resolvconf > /dev/null 2>&1 && systemctl restart resolvconf
+
+ which wget > /dev/null || (aptget_wrapper update; aptget_wrapper install -y wget)
+
+ # SUGGESTED UPDATE:
+ #export MASTER_IP="$config_host" MINION_ID="$node_hostname.$node_domain" SALT_VERSION=$saltversion
+ #source <(curl -qL ${BOOTSTRAP_SCRIPT_URL})
+ ## Update BOOTSTRAP_SALTSTACK_OPTS, as by default they contain "-dX" not to start service
+ #BOOTSTRAP_SALTSTACK_OPTS=" stable $SALT_VERSION "
+ #install_salt_minion_pkg
+
+ # DEPRECATED:
+ echo "deb [arch=amd64] http://apt-mk.mirantis.com/xenial ${DISTRIB_REVISION} salt extra" > /etc/apt/sources.list.d/mcp_salt.list
+ wget -O - http://apt-mk.mirantis.com/public.gpg | apt-key add -
+
+ echo "deb http://repo.saltstack.com/apt/ubuntu/16.04/amd64/$saltversion xenial main" > /etc/apt/sources.list.d/saltstack.list
+ wget -O - "https://repo.saltstack.com/apt/ubuntu/16.04/amd64/$saltversion/SALTSTACK-GPG-KEY.pub" | apt-key add -
+
+ aptget_wrapper clean
+ aptget_wrapper update
+ aptget_wrapper install -y salt-minion
+ ;;
+ rhel|centos|centos7|centos7|rhel6|rhel7)
+ yum install -y git
+ export MASTER_IP="$config_host" MINION_ID="$node_hostname.$node_domain" SALT_VERSION=$saltversion
+ source <(curl -qL ${BOOTSTRAP_SCRIPT_URL})
+ # Update BOOTSTRAP_SALTSTACK_OPTS, as by default they contain "-dX" not to start service
+ BOOTSTRAP_SALTSTACK_OPTS=" stable $SALT_VERSION "
+ install_salt_minion_pkg
+ ;;
+ *)
+ msg="OS '$node_os' is not supported."
+ esac
+
+ echo "Configuring Salt minion ..."
+ [ ! -d /etc/salt/minion.d ] && mkdir -p /etc/salt/minion.d
+ echo -e "id: $node_hostname.$node_domain\nmaster: $config_host" > /etc/salt/minion.d/minion.conf
+
+ service salt-minion restart || wait_condition_send "FAILURE" "Failed to restart salt-minion service."
+
+ sleep 1
+
+ echo "Classifying node ..."
+ os_codename=$(salt-call grains.item oscodename --out key | awk '/oscodename/ {print $2}')
+ node_network01_ip="$(ip a | awk -v prefix="^ inet $network01_prefix[.]" '$0 ~ prefix {split($2, a, "/"); print a[1]}')"
+ node_network02_ip="$(ip a | awk -v prefix="^ inet $network02_prefix[.]" '$0 ~ prefix {split($2, a, "/"); print a[1]}')"
+ node_network03_ip="$(ip a | awk -v prefix="^ inet $network03_prefix[.]" '$0 ~ prefix {split($2, a, "/"); print a[1]}')"
+ node_network04_ip="$(ip a | awk -v prefix="^ inet $network04_prefix[.]" '$0 ~ prefix {split($2, a, "/"); print a[1]}')"
+ node_network05_ip="$(ip a | awk -v prefix="^ inet $network05_prefix[.]" '$0 ~ prefix {split($2, a, "/"); print a[1]}')"
+
+ node_network01_iface="$(ip a | awk -v prefix="^ inet $network01_prefix[.]" '$0 ~ prefix {split($7, a, "/"); print a[1]}')"
+ node_network02_iface="$(ip a | awk -v prefix="^ inet $network02_prefix[.]" '$0 ~ prefix {split($7, a, "/"); print a[1]}')"
+ node_network03_iface="$(ip a | awk -v prefix="^ inet $network03_prefix[.]" '$0 ~ prefix {split($7, a, "/"); print a[1]}')"
+ node_network04_iface="$(ip a | awk -v prefix="^ inet $network04_prefix[.]" '$0 ~ prefix {split($7, a, "/"); print a[1]}')"
+ node_network05_iface="$(ip a | awk -v prefix="^ inet $network05_prefix[.]" '$0 ~ prefix {split($7, a, "/"); print a[1]}')"
+
+ if [ "$node_network05_iface" != "" ]; then
+ node_network05_hwaddress="$(cat /sys/class/net/$node_network05_iface/address)"
+ fi
+
+ # find more parameters (every env starting param_)
+ more_params=$(env | grep "^param_" | sed -e 's/=/":"/g' -e 's/^/"/g' -e 's/$/",/g' | tr "\n" " " | sed 's/, $//g')
+ if [ "$more_params" != "" ]; then
+ echo "Additional params: $more_params"
+ more_params=", $more_params"
+ fi
+
+ salt-call event.send "reclass/minion/classify" "{\"node_master_ip\": \"$config_host\", \"node_os\": \"${os_codename}\", \"node_deploy_ip\": \"${node_network01_ip}\", \"node_deploy_iface\": \"${node_network01_iface}\", \"node_control_ip\": \"${node_network02_ip}\", \"node_control_iface\": \"${node_network02_iface}\", \"node_tenant_ip\": \"${node_network03_ip}\", \"node_tenant_iface\": \"${node_network03_iface}\", \"node_external_ip\": \"${node_network04_ip}\", \"node_external_iface\": \"${node_network04_iface}\", \"node_baremetal_ip\": \"${node_network05_ip}\", \"node_baremetal_iface\": \"${node_network05_iface}\", \"node_baremetal_hwaddress\": \"${node_network05_hwaddress}\", \"node_domain\": \"$node_domain\", \"node_cluster\": \"$cluster_name\", \"node_hostname\": \"$node_hostname\"${more_params}}"
+
+ sleep 5
+
+ salt-call saltutil.sync_all
+ salt-call mine.flush
+ salt-call mine.update
+ node_name: {{ ssh['node_name'] }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
+{%- endif %}
+{%- endfor %}
+
+- description: Refresh pillars on all minions
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.refresh_pillar
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Sync all salt resources
+ cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.sync_all && sleep 5
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+{%- endmacro %}
\ No newline at end of file