Fix cicd and sl steps for virtual-ocata-cicd

Change-Id: I1cbc3cbe66d70beb91c05e849b6bbd7dbf632443
Reviewed-on: https://review.gerrithub.io/370564
Reviewed-by: Dennis Dmitriev <dis.xcom@gmail.com>
Tested-by: Dennis Dmitriev <dis.xcom@gmail.com>
diff --git a/tcp_tests/templates/virtual-mcp-ocata-cicd/oss.yaml b/tcp_tests/templates/virtual-mcp-ocata-cicd/oss.yaml
index 0528776..af678dc 100644
--- a/tcp_tests/templates/virtual-mcp-ocata-cicd/oss.yaml
+++ b/tcp_tests/templates/virtual-mcp-ocata-cicd/oss.yaml
@@ -130,7 +130,7 @@
   skip_fail: false
 
 - description: Install Docker client
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master' state.sls docker.client
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ CICD_NODES_SELECTOR }}' state.sls docker.client
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 5}
   skip_fail: false
@@ -145,7 +145,7 @@
 # Aptly
 #------
 
-- description: Install Aptly and check it's API
+- description: "Install Aptly and check it's API"
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@aptly:server' state.sls aptly &&
     timeout 90 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@aptly:server' cmd.run
       'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
@@ -157,7 +157,7 @@
 # OpenLDAP
 #---------
 
-- description: Install LDAP and check it's availability
+- description: "Install LDAP and check it's availability"
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@openldap:client' state.sls openldap &&
     timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@openldap:client' cmd.run
       'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
@@ -170,7 +170,7 @@
 # Gerrit
 #-------
 
-- description: Install Gerrit and check it's availability
+- description: "Install Gerrit and check it's availability"
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@gerrit:client' state.sls gerrit &&
     timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@gerrit:client' cmd.run
       'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
@@ -183,13 +183,16 @@
 #--------
 
 
-- description: Install Jenkins and check it's availability
-  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' state.sls jenkins
-  # TODO: Jenkins requires authenticate token. Remove -f from curl for now.
+- description: "Install Jenkins and check it's availability"
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' state.sls jenkins &&
     timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' cmd.run
       'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
-       while true; do curl -s http://${CICD_CONTROL_ADDRESS}:8081 >/dev/null && break; sleep 2; done'
+       export JENKINS_CLIENT_USER=$(salt-call --out=newline_values_only pillar.get _param:jenkins_client_user);
+       export JENKINS_CLIENT_PASSWORD=$(salt-call --out=newline_values_only pillar.get _param:jenkins_client_password);
+       while true; do
+         curl -f -u ${JENKINS_CLIENT_USER}:${JENKINS_CLIENT_PASSWORD} http://${CICD_CONTROL_ADDRESS}:8081/api/json?pretty=true && break;
+         sleep 2;
+       done'
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
@@ -211,7 +214,7 @@
 - description: Check Pushkin API availability
   cmd: timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@postgresql:client' cmd.run
     'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
-     while true; do curl -s http://${CICD_CONTROL_ADDRESS}:8887 && break; sleep 2; done'
+     while true; do curl -sf http://${CICD_CONTROL_ADDRESS}:8887/apps && break; sleep 2; done'
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp-ocata-cicd/salt.yaml b/tcp_tests/templates/virtual-mcp-ocata-cicd/salt.yaml
index 942c947..fc38e66 100644
--- a/tcp_tests/templates/virtual-mcp-ocata-cicd/salt.yaml
+++ b/tcp_tests/templates/virtual-mcp-ocata-cicd/salt.yaml
@@ -274,9 +274,16 @@
   retry: {count: 1, delay: 5}
   skip_fail: false
 
-- description: Configure minion on master
+- description: Configure salt.minion on master
   cmd: salt --hard-crash --state-output=mixed --state-verbose=False
-    -C 'I@salt:master' state.sls 'salt.minion'
+    -C 'I@salt:master' state.sls salt.minion
+  node_name: {{ HOSTNAME_CFG01 }}
+  retry: {count: 3, delay: 10}
+  skip_fail: false
+
+- description: Run state 'salt' on master (for salt.api, etc)
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
+    -C 'I@salt:master' state.sls salt
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 3, delay: 10}
   skip_fail: false
diff --git a/tcp_tests/templates/virtual-mcp-ocata-cicd/sl.yaml b/tcp_tests/templates/virtual-mcp-ocata-cicd/sl.yaml
index 3a89bf2..7e0faf3 100644
--- a/tcp_tests/templates/virtual-mcp-ocata-cicd/sl.yaml
+++ b/tcp_tests/templates/virtual-mcp-ocata-cicd/sl.yaml
@@ -1,51 +1,44 @@
 {% from 'virtual-mcp-ocata-cicd/underlay.yaml' import HOSTNAME_CFG01 with context %}
 
+# Pattern that helps salt to select Stacklight nodes
+{% set SL_NODES_SELECTOR = os_env('SL_NODES_SELECTOR','mon*') %}
+
 # Install docker swarm
-- description: Configure docker service
-  cmd: salt -C 'I@docker:swarm' state.sls docker.host
+- description: Prepare Docker host
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:host and {{ SL_NODES_SELECTOR }}' state.sls docker.host
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 1, delay: 5}
   skip_fail: false
 
-- description: Install docker swarm on master node
-  cmd: salt -C 'I@docker:swarm:role:master' state.sls docker.swarm
+- description: Install Docker Swarm master
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ SL_NODES_SELECTOR }}' state.sls docker.swarm
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 1, delay: 5}
   skip_fail: false
 
-- description: Send grains to the swarm slave nodes
-  cmd: salt -C 'I@docker:swarm' state.sls salt.minion.grains
+- description: Setup salt-minion on docker swarm master
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ SL_NODES_SELECTOR }}' state.sls salt &&
+    salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ SL_NODES_SELECTOR }}' mine.flush &&
+    salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ SL_NODES_SELECTOR }}' mine.update
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 3, delay: 15}
   skip_fail: false
 
-- description:  Update mine
-  cmd: salt -C 'I@docker:swarm' mine.update
+- description: Install Docker Swarm on other nodes
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and {{ SL_NODES_SELECTOR }}' state.sls docker.swarm
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 1, delay: 5}
   skip_fail: false
 
-- description:  Refresh modules
-  cmd: salt -C 'I@docker:swarm' saltutil.refresh_modules; sleep 5;
+- description: Show Docker Swarm nodes
+  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ SL_NODES_SELECTOR }}' cmd.run 'docker node ls'
   node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
-  skip_fail: false
-
-- description:  Configure slave nodes
-  cmd: salt -C 'I@docker:swarm:role:manager' state.sls docker.swarm -b 1
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
-  skip_fail: false
-
-- description:  List registered Docker swarm nodes
-  cmd: salt -C 'I@docker:swarm:role:master' cmd.run 'docker node ls'
-  node_name: {{ HOSTNAME_CFG01 }}
-  retry: {count: 1, delay: 10}
+  retry: {count: 1, delay: 5}
   skip_fail: false
 
 # Install slv2 infra
 - description: Install telegraf
-  cmd: salt -C 'I@telegraf:agentor or I@telegraf:remote_agent' state.sls telegraf
+  cmd: salt -C 'I@telegraf:agent or I@telegraf:remote_agent' state.sls telegraf
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
@@ -125,19 +118,19 @@
 
 # Configure the services running in Docker Swarm
 - description: Install prometheus alertmanager
-  cmd: salt -C 'I@docker:swarm' state.sls prometheus,heka.remote_collector -b 1
+  cmd: salt -C 'I@docker:swarm and {{ SL_NODES_SELECTOR }}' state.sls prometheus,heka.remote_collector -b 1
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
 - description: run docker state
-  cmd: salt -C 'I@docker:swarm:role:master' state.sls docker
+  cmd: salt -C 'I@docker:swarm:role:master and {{ SL_NODES_SELECTOR }}' state.sls docker
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false
 
 - description: docker ps
-  cmd: salt -C 'I@docker:swarm' dockerng.ps
+  cmd: salt -C 'I@docker:swarm and {{ SL_NODES_SELECTOR }}' dockerng.ps
   node_name: {{ HOSTNAME_CFG01 }}
   retry: {count: 1, delay: 10}
   skip_fail: false