Working deploy for controller and compute nodes

- use 'dash' branch for mk-lab-salt-model repo (it has a lot of
  improvements and fixes)
- use less number of nodes to match the 'mk22-lab-advanced' model
- force update packages before configuring salt.minions
- use saltstack repo for latest salt.minions on trusty

* Note if the deploy starts from an 'underlay' snapshot,
  keepalived can lose VIP after update DHCP leases on nodes.
  It's better to start deploy from clean environment.
diff --git a/tcp_tests/settings_oslo.py b/tcp_tests/settings_oslo.py
index 7e8490d..6ba5b3c 100644
--- a/tcp_tests/settings_oslo.py
+++ b/tcp_tests/settings_oslo.py
@@ -42,6 +42,7 @@
     ct.Cfg('ssh', ct.JSONList(),
            help="""SSH Settings for Underlay: [{
                   'node_name': node1,
+                  'roles': ['salt-master', 'salt-minion', ],
                   'host': hostname,
                   'login': login,
                   'password': password,
diff --git a/tcp_tests/templates/tcpcloud--user-data-master-node.yaml b/tcp_tests/templates/tcpcloud--user-data-master-node.yaml
index bfbb85f..fa24840 100644
--- a/tcp_tests/templates/tcpcloud--user-data-master-node.yaml
+++ b/tcp_tests/templates/tcpcloud--user-data-master-node.yaml
@@ -39,6 +39,8 @@
    - which wget >/dev/null || (apt-get update; apt-get install -y wget)
 
    - echo "deb [arch=amd64] http://apt.tcpcloud.eu/nightly/ xenial main security extra tcp tcp-salt" > /etc/apt/sources.list
+   # 'tcp-salt' from trusty is for temporary workaround until formulas will be fixed in xenial
+   - echo "deb [arch=amd64] http://apt.tcpcloud.eu/nightly/ trusty tcp-salt" >> /etc/apt/sources.list
    - wget -O - http://apt.tcpcloud.eu/public.gpg | apt-key add -
 
    - apt-get clean
@@ -77,7 +79,17 @@
    - HOME=/root git config --global user.name "TCP QA"
 
    - echo "Configuring reclass ..."
-   - git clone https://github.com/Mirantis/mk-lab-salt-model.git /srv/salt/reclass -b master
+   - git clone https://github.com/Mirantis/mk-lab-salt-model.git /srv/salt/reclass -b dash
+   - sed -i 's/ master/ dash/' /srv/salt/reclass/classes/cluster/mk20_lab_advanced/openstack_config.yml
+   - |
+       cat << 'EOF' >> /srv/salt/reclass/nodes/control/{hostname}.yml
+       # local storage
+         reclass:
+           storage:
+             data_source:
+               engine: local
+       EOF
+   - cd /srv/salt/reclass; git add -A;git commit -m"use dash repo"
 
    - mkdir -p /srv/salt/reclass/classes/service
    - for i in /usr/share/salt-formulas/reclass/service/*; do ln -s $i /srv/salt/reclass/classes/service/; done
diff --git a/tcp_tests/templates/tcpcloud--user-data.yaml b/tcp_tests/templates/tcpcloud--user-data.yaml
index d60a87a..ecde655 100644
--- a/tcp_tests/templates/tcpcloud--user-data.yaml
+++ b/tcp_tests/templates/tcpcloud--user-data.yaml
@@ -41,6 +41,9 @@
 
    - echo "deb [arch=amd64] http://apt.tcpcloud.eu/nightly/ trusty main security extra tcp tcp-salt" > /etc/apt/sources.list
    - wget -O - http://apt.tcpcloud.eu/public.gpg | apt-key add -
+   # saltstack repo is for minions that have the same version in the xenial and trusty (2016.3.3)
+   - echo "deb http://repo.saltstack.com/apt/ubuntu/14.04/amd64/latest trusty main" > /etc/apt/sources.list.d/saltstack.list
+   - wget -O - https://repo.saltstack.com/apt/ubuntu/14.04/amd64/latest/SALTSTACK-GPG-KEY.pub | apt-key add -
 
    - apt-get clean
    - apt-get update
diff --git a/tcp_tests/templates/tcpcloud-default.yaml b/tcp_tests/templates/tcpcloud-default.yaml
index 0a8cbdc..597a759 100644
--- a/tcp_tests/templates/tcpcloud-default.yaml
+++ b/tcp_tests/templates/tcpcloud-default.yaml
@@ -21,10 +21,8 @@
             default_ctl01.mk22-lab-advanced.local: +101
             default_ctl02.mk22-lab-advanced.local: +102
             default_ctl03.mk22-lab-advanced.local: +103
-            default_web01.mk22-lab-advanced.local: +104
+            default_prx01.mk22-lab-advanced.local: +104
             default_cmp01.mk22-lab-advanced.local: +105
-            default_mtr01.mk22-lab-advanced.local: +106
-            default_mon01.mk22-lab-advanced.local: +107
           ip_ranges:
             dhcp: [+100, -2]
 
@@ -227,7 +225,7 @@
                   networks:
                     - private
 
-          - name: web01.mk22-lab-advanced.local
+          - name: prx01.mk22-lab-advanced.local
             role: salt_minion
             params:
               vcpu: !os_env SLAVE_NODE_CPU, 1
@@ -302,79 +300,3 @@
                 eth1:
                   networks:
                     - private
-
-          - name: mtr01.mk22-lab-advanced.local
-            role: salt_minion
-            params:
-              vcpu: !os_env SLAVE_NODE_CPU, 1
-              memory: !os_env SLAVE_NODE_MEMORY, 2048
-              boot:
-                - hd
-              cloud_init_volume_name: iso
-              cloud_init_iface_up: eth0
-              volumes:
-                - name: system
-                  capacity: !os_env NODE_VOLUME_SIZE, 150
-                  backing_store: cloudimage1404
-                  format: qcow2
-                - name: iso  # Volume with name 'iso' will be used
-                             # for store image with cloud-init metadata.
-                  capacity: 1
-                  format: raw
-                  device: cdrom
-                  bus: ide
-                  cloudinit_meta_data: !include tcpcloud--meta-data.yaml
-                  cloudinit_user_data: !include tcpcloud--user-data.yaml
-
-              interfaces:
-                - label: eth0
-                  l2_network_device: public
-                  interface_model: *interface_model
-                - label: eth1
-                  l2_network_device: private
-                  interface_model: *interface_model
-              network_config:
-                eth0:    # Will get an IP from DHCP public-pool01
-                  networks:
-                    - public
-                eth1:
-                  networks:
-                    - private
-
-          - name: mon01.mk22-lab-advanced.local
-            role: salt_minion
-            params:
-              vcpu: !os_env SLAVE_NODE_CPU, 1
-              memory: !os_env SLAVE_NODE_MEMORY, 2048
-              boot:
-                - hd
-              cloud_init_volume_name: iso
-              cloud_init_iface_up: eth0
-              volumes:
-                - name: system
-                  capacity: !os_env NODE_VOLUME_SIZE, 150
-                  backing_store: cloudimage1404
-                  format: qcow2
-                - name: iso  # Volume with name 'iso' will be used
-                             # for store image with cloud-init metadata.
-                  capacity: 1
-                  format: raw
-                  device: cdrom
-                  bus: ide
-                  cloudinit_meta_data: !include tcpcloud--meta-data.yaml
-                  cloudinit_user_data: !include tcpcloud--user-data.yaml
-
-              interfaces:
-                - label: eth0
-                  l2_network_device: public
-                  interface_model: *interface_model
-                - label: eth1
-                  l2_network_device: private
-                  interface_model: *interface_model
-              network_config:
-                eth0:    # Will get an IP from DHCP public-pool01
-                  networks:
-                    - public
-                eth1:
-                  networks:
-                    - private
diff --git a/tcp_tests/tests/system/test_tcp_install.py b/tcp_tests/tests/system/test_tcp_install.py
index 685d3e9..f9ab672 100644
--- a/tcp_tests/tests/system/test_tcp_install.py
+++ b/tcp_tests/tests/system/test_tcp_install.py
@@ -29,13 +29,13 @@
 
     #salt_cmd = 'salt -l debug '  # For debug output
     #salt_call_cmd = 'salt-call -l debug '  # For debug output
-    salt_cmd = 'salt --state-output=mixed --state-verbose=False '  # For cause only output
-    salt_call_cmd = 'salt-call --state-output=mixed --state-verbose=False '  # For cause only output
+    salt_cmd = 'salt --hard-crash --state-output=mixed --state-verbose=False '  # For cause only output
+    salt_call_cmd = 'salt-call --hard-crash --state-output=mixed --state-verbose=False '  # For cause only output
     #salt_cmd = 'salt --state-output=terse --state-verbose=False '  # For reduced output
     #salt_call_cmd = 'salt-call --state-output=terse --state-verbose=False '  # For reduced output
 
-
-    steps_mk22_lab_advanced = [
+    steps_mk22_advanced_lab = [
+        # Prepare salt services and nodes settings
         {
             'description': "Run 'linux' formula on cfg01",
             'cmd': salt_call_cmd + "state.sls linux",
@@ -90,6 +90,27 @@
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+
+
+        # Bootstrap all nodes
+        {
+            'description': "Configure linux on controllers",
+            'cmd': salt_cmd + "'*' state.sls linux",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 5, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Configure openssh on controllers",
+            'cmd': (salt_cmd + "-C '* and not cfg*' state.sls openssh;"
+                    + salt_cmd + "-C '* and not cfg*' cmd.run "
+                    "\"sed -i 's/PasswordAuthentication no/"
+                    "PasswordAuthentication yes/' /etc/ssh/sshd_config && "
+                    "service ssh restart\""),
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
         {
             'description': ("*Workaround* for the bug"
                             " https://mirantis.jira.com/browse/PROD-8025"),
@@ -109,38 +130,30 @@
             'skip_fail': False,
         },
         {
-            'description': "Configure ntp on controllers",
-            'cmd': salt_cmd + "'ctl*' state.sls ntp",
-            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 5, 'delay': 10},
-            'skip_fail': False,
-        },
-        {
-            'description': "Configure linux on controllers",
-            'cmd': salt_cmd + "'ctl*' state.sls linux",
-            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 5, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Configure openssh on controllers",
-            'cmd': (salt_cmd + "'ctl*' state.sls openssh;"
-                    + salt_cmd + "'ctl*' cmd.run "
-                    "\"sed -i 's/PasswordAuthentication no/"
-                    "PasswordAuthentication yes/' /etc/ssh/sshd_config && "
-                    "service ssh restart\""),
+            'description': ("*Workaround* for the bug"
+                            " https://mirantis.jira.com/browse/PROD-XXXXX"),
+            'cmd': (salt_cmd + "'*' cmd.run 'apt-get -y install python-requests'"),
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
             'description': "Configure salt.minion on controllers",
-            'cmd': salt_cmd + "'ctl*' state.sls salt.minion",
+            'cmd': salt_cmd + "'*' state.sls salt.minion",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
+            'description': "Configure ntp on controllers",
+            'cmd': salt_cmd + "'*' state.sls ntp",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 5, 'delay': 10},
+            'skip_fail': False,
+        },
+
+        # Install support services
+        {
             'description': "Install keepalived on primary controller",
             'cmd': salt_cmd + "'ctl01*' state.sls keepalived",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
@@ -148,19 +161,25 @@
             'skip_fail': False,
         },
         {
-            'description': "Show VIP on primary controller",
-            'cmd': salt_cmd + "'ctl01*' cmd.run 'ip a'",
+            'description': "Install keepalived on other controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls keepalived -b 1",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Install keepalived on other controllers",
-            'cmd': salt_cmd + "'ctl0[23].*' state.sls keepalived",
+            'description': "Check the VIP",
+            # First grep finds the IP, second is to get the correct exit code
+            'cmd': (salt_cmd + "'ctl*' cmd.run 'ip a | grep 172.16.10.254' |"
+                    " grep -B1 172.16.10.254"),
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+
+
+
+
         {
             'description': "Install glusterfs on all controllers",
             'cmd': salt_cmd + "'ctl*' state.sls glusterfs.server.service",
@@ -170,25 +189,29 @@
         },
         {
             'description': "Setup glusterfs on primary controller",
-            'cmd': salt_call_cmd + "state.sls glusterfs.server.setup",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'cmd': salt_cmd + "'ctl01*' state.sls glusterfs.server.setup",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Show glusterfs peer status",
-            'cmd': "gluster peer status",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Setup glusterfs on other controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls glusterfs.server.setup -b 1",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Show glusterfs volume status",
-            'cmd': "gluster volume status",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Check the gluster status",
+            'cmd': salt_cmd + "'ctl01*' cmd.run 'gluster peer status; gluster volume status'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+
+
+
+
         {
             'description': "Install RabbitMQ on all controllers",
             'cmd': salt_cmd + "'ctl*' state.sls rabbitmq",
@@ -197,6 +220,16 @@
             'skip_fail': False,
         },
         {
+            'description': "Check the rabbitmq status",
+            'cmd': salt_cmd + "'ctl*' cmd.run 'rabbitmqctl cluster_status'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+
+
+
+        {
             'description': ("*Workaround* Update salt-formula-galera on"
                             " config node to the latest version"),
             'cmd': "apt-get -y --force-yes install salt-formula-galera",
@@ -213,19 +246,22 @@
         },
         {
             'description': "Install Galera on other controllers",
-            'cmd': salt_cmd + "'ctl0[23]*' state.sls galera",
+            'cmd': salt_cmd + "'ctl*' state.sls galera",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Check Galera addresses",
-            'cmd': (salt_cmd + "'ctl01*'  mysql.status |"
-                    " grep -A1 'wsrep_incoming_addresses:'"),
+            'description': "Check mysql status",
+            'cmd': salt_cmd + "'ctl*' mysql.status",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
+            'skip_fail': True,
         },
+
+
+
+
         {
             'description': "Install haproxy on all controllers",
             'cmd': salt_cmd + "'ctl*' state.sls haproxy",
@@ -234,167 +270,180 @@
             'skip_fail': False,
         },
         {
-            'description': "Check haproxy on all controllers with Galera port",
-            'cmd': salt_cmd + "'ctl*' cmd.run 'netstat -tulnp | grep 3306'",
+            'description': "Check haproxy status",
+            'cmd': salt_cmd + "'ctl*' service.status haproxy",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+
+
+        {
+            'description': "Install memcached on all controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls memcached",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+
+
+        # Install OpenStack control services
+
+
+        {
+            'description': "Install keystone on primary controller",
+            'cmd': salt_cmd + "'ctl01*' state.sls keystone",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Install memcached and keystone on ctl01",
-            'cmd': salt_call_cmd + "state.sls memcached,keystone",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Install keystone on all controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls keystone -b 1",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Install memcached and keystone on ctl02",
-            'cmd': salt_call_cmd + "state.sls memcached,keystone",
-            'node_name': 'ctl02.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Install memcached and keystone on ctl03",
-            'cmd': salt_call_cmd + "state.sls memcached,keystone",
-            'node_name': 'ctl03.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Configure keystone client access from cfg01 node",
+            'description': "Populate keystone services/tenants/admins",
             'cmd': salt_call_cmd + "state.sls keystone.client",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Check keystone user-list",
-            'cmd': "source ~/keystonerc; keystone user-list",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Check keystone service-list",
+            'cmd': salt_cmd + "'ctl01*' cmd.run '. /root/keystonerc; keystone service-list'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+
+
+
         {
-            'description': "Check keystone tenant-list",
-            'cmd': "source ~/keystonerc; keystone tenant-list",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Check keystone endpoint-list",
-            'cmd': "source ~/keystonerc; keystone endpoint-list",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Install glance on controllers",
-            'cmd': salt_cmd + "'ctl*' state.sls glance",
+            'description': "Install glance on primary controller",
+            'cmd': salt_cmd + "'ctl01*' state.sls glance",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Install glusterfs on controllers",
+            'description': "Install glance on all controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls glance -b 1",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Configure glusterfs.client on all controllers",
             'cmd': salt_cmd + "'ctl*' state.sls glusterfs.client",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Check that glusterfs was added on controllers",
-            'cmd': salt_cmd + "'ctl*' cmd.run 'df -h'",
+            'description': "Configure(re-install) keystone on all controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls keystone -b 1",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': ("*Workaround* Re-run keystone formula on ctl01 to"
-                            " create fernet keys"),
-            'cmd': salt_call_cmd + "state.sls keystone",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Check glance image-list",
+            'cmd': salt_cmd + "'ctl01*' cmd.run '. /root/keystonerc; glance image-list'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+
+
+
         {
-            'description': "Check glance on ctl01",
-            'cmd': ("source ~/keystonerc;"
-                    "wget http://download.cirros-cloud.net/0.3.4/"
-                    "cirros-0.3.4-i386-disk.img;"
-                    "glance image-create --name 'cirros-0.3.4'"
-                    "  --disk-format qcow2 --container-format bare"
-                    "  --progress --file /root/cirros-0.3.4-i386-disk.img;"
-                    "glance image-list;"),
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Check keystone fernet keys on controllers",
-            'cmd': (salt_cmd + "'ctl*' cmd.run 'ls -la"
-                    " /var/lib/keystone/fernet-keys' "),
+            'description': "Install cinder on all controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls cinder -b 1",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Install cinder on controllers",
-            'cmd': salt_cmd + "'ctl*' state.sls cinder",
+            'description': "Check cinder list",
+            'cmd': salt_cmd + "'ctl01*' cmd.run '. /root/keystonerc; cinder list'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+
+
+        {
+            'description': "Install nova on ctl01",
+            'cmd': salt_cmd + "'ctl01*' state.sls nova",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Install nova on controllers",
+            'description': "Install nova on all controllers",
             'cmd': salt_cmd + "'ctl*' state.sls nova",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Check cinder status",
-            'cmd': "source ~/keystonerc; cinder list",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Check nova service-list",
+            'cmd': salt_cmd + "'ctl01*' cmd.run '. /root/keystonerc; nova service-list'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+
+
+        {
+            'description': "Install neutron on ctl01",
+            'cmd': salt_cmd + "'ctl01*' state.sls neutron",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Check nova services status",
-            'cmd': "source ~/keystonerc; nova-manage service list",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Check nova status",
-            'cmd': "source ~/keystonerc; nova list",
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
-            'retry': {'count': 3, 'delay': 5},
-            'skip_fail': False,
-        },
-        {
-            'description': "Install neutron on controllers",
+            'description': "Install neutron on all controllers",
             'cmd': salt_cmd + "'ctl*' state.sls neutron",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
         {
-            'description': "Create a neutron subnet",
-            'cmd': ("source ~/keystonerc;"
-                    "neutron net-create --router:external=true"
-                    " --shared external;"
-                    "neutron subnet-create external 10.177.0.0/24;"
-                    "neutron floatingip-create;"),
-            'node_name': 'ctl01.mk22-lab-advanced.local',  # hardcoded for now
+            'description': "Check neutron agent-list",
+            'cmd': salt_cmd + "'ctl01*' cmd.run '. /root/keystonerc; neutron agent-list'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+
+
+        {
+            'description': "Deploy dashboard on prx*",
+            'cmd': salt_cmd + "'prx*' state.apply",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': True,
+        },
+
+
+        {
+            'description': "Deploy nginx proxy",
+            'cmd': salt_cmd + "'cfg*' state.sls nginx",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': True,
+        },
+
+
+
+        # Install contrail on controllers
         {
             'description': "Install contrail database on controllers",
-            'cmd': salt_cmd + "'ctl*' state.sls opencontrail.database",
+            'cmd': salt_cmd + "'ctl*' state.sls opencontrail.database -b 1",
             'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
@@ -408,10 +457,134 @@
             'retry': {'count': 3, 'delay': 5},
             'skip_fail': False,
         },
+        {
+            'description': "Install contrail database on controllers",
+            'cmd': salt_cmd + "'ctl*' state.sls opencontrail -b 1",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Check contrail status",
+            'cmd': (salt_cmd + "'ctl01*' cmd.run '. /root/keystonerc;"
+                    " contrail-status; neutron net-list; nova net-list'"),
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Add contrail bgp router on ctl01",
+            'cmd': (salt_cmd + "'ctl01*' cmd.run "
+                    "'/usr/share/contrail-utils/provision_control.py"
+                    " --oper add"
+                    " --api_server_ip 172.16.10.254"
+                    " --api_server_port 8082"
+                    " --host_name ctl01"
+                    " --host_ip 172.16.10.101"
+                    " --router_asn 64512"
+                    " --admin_user admin"
+                    " --admin_password workshop"
+                    " --admin_tenant_name admin"
+                    "'"),
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Add contrail bgp router on ctl02",
+            'cmd': (salt_cmd + "'ctl02*' cmd.run "
+                    "'/usr/share/contrail-utils/provision_control.py"
+                    " --oper add"
+                    " --api_server_ip 172.16.10.254"
+                    " --api_server_port 8082"
+                    " --host_name ctl02"
+                    " --host_ip 172.16.10.102"
+                    " --router_asn 64512"
+                    " --admin_user admin"
+                    " --admin_password workshop"
+                    " --admin_tenant_name admin"
+                    "'"),
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Add contrail bgp router on ctl03",
+            'cmd': (salt_cmd + "'ctl03*' cmd.run "
+                    "'/usr/share/contrail-utils/provision_control.py"
+                    " --oper add"
+                    " --api_server_ip 172.16.10.254"
+                    " --api_server_port 8082"
+                    " --host_name ctl03"
+                    " --host_ip 172.16.10.103"
+                    " --router_asn 64512"
+                    " --admin_user admin"
+                    " --admin_password workshop"
+                    " --admin_tenant_name admin"
+                    "'"),
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+
+
+        # Install compute node
+        {
+            'description': "Apply formulas for compute node",
+            'cmd': salt_cmd + "'cmp*' state.apply",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Re-apply(as in doc) formulas for compute node",
+            'cmd': salt_cmd + "'cmp*' state.apply",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Add vrouter for cmp01",
+            'cmd': (salt_cmd + "'ctl01*' cmd.run "
+                    "'/usr/share/contrail-utils/provision_vrouter.py"
+                    " --oper add"
+                    " --host_name cmp01"
+                    " --host_ip 172.16.10.105"
+                    " --api_server_ip 172.16.10.254"
+                    " --api_server_port 8082"
+                    " --admin_user admin"
+                    " --admin_password workshop"
+                    " --admin_tenant_name admin"
+                    "'"),
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Reboot compute nodes",
+            'cmd': salt_cmd + "'cmp*' system.reboot",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Check IP on computes",
+            'cmd': salt_cmd + "'cmp*' cmd.run 'ip a'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
+        {
+            'description': "Check contrail status on computes",
+            'cmd': salt_cmd + "'cmp*' cmd.run 'contrail-status'",
+            'node_name': 'cfg01.mk22-lab-advanced.local',  # hardcoded for now
+            'retry': {'count': 3, 'delay': 5},
+            'skip_fail': False,
+        },
     ]
 
 
-    @pytest.mark.steps(steps_mk22_lab_advanced)
+    @pytest.mark.steps(steps_mk22_advanced_lab)
     @pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
     # @pytest.mark.snapshot_needed
     # @pytest.mark.fail_snapshot