Update local kitchen-salt, model validation scripts

Change-Id: I98883734817044e8aa599d745ca6c57af2d1b999
diff --git a/.kitchen-verify.sh b/.kitchen-verify.sh
new file mode 100755
index 0000000..8db3819
--- /dev/null
+++ b/.kitchen-verify.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+## -----------------------------------------------------------------------------
+# PREREQ
+if [ -e /tmp/kitchen ]; then
+  rsync -avh --force --exclude "**/.kitchen" /tmp/kitchen/ /srv/salt/reclass
+fi
+source /srv/salt/bootstrap.sh
+source_local_envs
+
+# FULL BOOTSTRAP
+# make sure everything is set-up and exactly as in the model
+if [[ $BOOTSTRAP =~ ^(True|true|1|yes)$ ]]; then
+  cd /srv/salt/reclass
+  if [[ -e .gitmodules ]] && [[ ! -e classes/system/linux ]]; then
+    git submodule update --init --recursive --remote || true
+  fi
+  /srv/salt/bootstrap.sh &&\
+  if [[ -e /tmp/kitchen ]]; then sed -i '/BOOTSTRAP=/d' /kitchen.env; fi || exit 1
+else
+# JUST CONFIGURE AND START SALT-MASTER
+# minimal setup to speed up, good-enough for model validation
+	mkdir -p /srv/salt/reclass/nodes/_generated;
+	cat <<-EOF > /srv/salt/reclass/nodes/_generated/cfg01.$CLUSTER_NAME.local.yml
+	classes:
+	- cluster.$CLUSTER_NAME.infra.config
+	parameters:
+	  _param:
+	    linux_system_codename: xenial
+	    reclass_data_revision:
+	    reclass_data_repository:
+	    cluster_name: $CLUSTER_NAME
+	    cluster_domain: local
+	  linux:
+	    system:
+	      name: cfg01
+	      domain: $CLUSTER_NAME.local
+	  reclass:
+	    storage:
+	      data_source:
+	        engine: local
+	EOF
+# RECLASS PREREQUISITES
+# remove lock if you need (re)configure salt-master, reclass 2nd time
+  if [ ! -e /tmp/.reclass-prerequisite-once.lock ]; then
+    # link services (subject of future changes)
+    if [ ! -e /srv/salt/reclass/classes/service -a -e /usr/share/salt-formulas/reclass/service ]; then
+      rm -rf /srv/salt/reclass/classes/service || true;
+      ln -sf /usr/share/salt-formulas/reclass/service /srv/salt/reclass/classes
+    fi &&\
+    # reload
+    service salt-master force-reload &&\
+    service salt-minion force-reload &&\
+    retry salt-call saltutil.sync_all &&\
+    # generate static nodes
+    PILLAR='{"reclass":{"storage":{"data_source":{"engine":"local"}}} }' &&\
+    salt-call state.apply reclass.storage.node pillar="$PILLAR" &&\
+    # generate/mockup dynamic nodes (uses fn from bootstrap.sh)
+    mockup_node_registration &&\
+    touch /tmp/.salt-services-reload-once.lock || exit 1
+  fi
+fi
+
+
+
+## -----------------------------------------------------------------------------
+## VERIFY MASTER AND MINIONS
+
+if [[ ! -e /tmp/.verify-prerequisite-once.lock && ! $RECLASS_IGNORE_CLASS_NOTFOUND =~ ^(True|true|1|yes)$ ]]; then
+  sed -i 's/ignore_class_notfound:\s*True.*/ignore_class_notfound: False/' /etc/salt/master.d/reclass.conf
+  sed -i 's/ignore_class_notfound:\s*True.*/ignore_class_notfound: False/' /etc/reclass/reclass-config.yml
+  service salt-master force-reload
+  retry salt-call test.ping
+fi
+
+cd /srv/salt/reclass &&\
+if [[ -z "$1" ]] ; then
+  verify_salt_master &&\
+  verify_salt_minions
+else
+  verify_salt_minion "$1"
+fi
diff --git a/.kitchen.yml b/.kitchen.yml
index a8c4051..9fee5b8 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -7,35 +7,41 @@
 
 provisioner:
   name: shell
-  script: verify.sh
+  script: .kitchen-verify.sh
+
+verifier:
+  name: inspec
+
+# Use: find classes/cluster/$cluster -name '*.env' | xargs -n1 source
+# TODO, Load and set ENV from **/*.env
+<%- saltver = ENV['SALT_VERSION'] || '2017.7' %>
+<%- btstrap = ENV['SALT_BOOTSTRAP'] || '0' %>
+<%- extrafo = ENV['EXTRA_FORMULAS'] || 'ntp' %>
+<%- formula = ENV['FORMULA_VERSION'] || 'master' %>
+<%- reclass = ENV['RECLASS_VERSION'] || 'develop' %>
 
 platforms:
   <% `find classes/cluster -maxdepth 1 -mindepth 1 -type d | tr '_' '-' |sort -u`.split().each do |cluster| %>
   <% cluster=cluster.split('/')[2] %>
   - name: <%= cluster %>
     driver_config:
-      #image: ubuntu:16.04
-      image: tcpcloud/salt-models-testing # With preinstalled dependencies (faster)
+      # image: epcim/salt:saltstack-ubuntu-xenial-salt-stable
+      # FIXME, USE ONCE RECLASS@DEVELOP IS IN MASTER, AND REQUIREDE FEATURES FROM FORMULA NIGHTLY REPO ARE IN STABLE
+      image: epcim/salt:saltmaster-reclass-ubuntu-xenial-salt-<%= saltver %>-formula-<%= formula %>
       platform: ubuntu
       hostname: cfg01.<%= cluster %>.local
       provision_command:
-        - apt-get update
-        - apt-get install -y git curl python-pip
-        - pip install --upgrade pip
-        - git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts
-        - cd /srv/salt/scripts; git pull -r; cd -
-        # NOTE: Configure ENV options as needed, example:
+        - test -e /srv/salt/bootstrap.sh || curl -qsSL https://raw.githubusercontent.com/salt-formulas/salt-formulas-scripts/develop/bootstrap.sh -o /srv/salt/bootstrap.sh
+        # Configure ENV options as needed, example:
         - echo "
-            export BOOTSTRAP=1;\n
+            export BOOTSTRAP=<%= btstrap %>;\n
             export CLUSTER_NAME=<%= cluster %>;\n
-            export FORMULAS_SOURCE=pkg;\n
-            export RECLASS_VERSION=master;\n
+            export EXTRA_FORMULAS='<%= extrafo %>';\n
+            export RECLASS_VERSION=<%= reclass %>;\n
             export RECLASS_IGNORE_CLASS_NOTFOUND=True;\n
             export RECLASS_IGNORE_CLASS_REGEXP='service.*';\n
-            export EXTRA_FORMULAS="";\n
           " > /kitchen.env
-          #export RECLASS_SOURCE_PATH=/usr/lib/python2.7/site-packages/reclass;\n
-          #export PYTHONPATH=$RECLASS_SOURCE_PATH:$PYTHONPATH;\n
+          #export FORMULAS_SOURCE=pkg;\n
   <% end %>
 
 suites:
diff --git a/README.rst b/README.rst
index cc8cc13..c9aabc6 100755
--- a/README.rst
+++ b/README.rst
@@ -159,8 +159,12 @@
   cluster-virtual-mcp-ocata-dvr             Docker  Shell        Busser    Ssh        <Not Created>  <None>
   cluster-virtual-mcp-ocata-ovs             Docker  Shell        Busser    Ssh        <Not Created>  <None>
 
+
+Example usage
+
 .. code-block:: shell
 
-  ➜  kitchen converge aaa-ha-freeipa
+  ➜  kitchen converge cluster-k8s-ha-calico
+  ➜  kitchen verify   cluster-os-ha-contrail
 
 
diff --git a/test/integration/cluster/01_salt_highstate_spec.rb b/test/integration/cluster/01_salt_highstate_spec.rb
new file mode 100644
index 0000000..7666571
--- /dev/null
+++ b/test/integration/cluster/01_salt_highstate_spec.rb
@@ -0,0 +1,11 @@
+
+describe command('salt-call pillar.items') do
+  its('exit_status') { should eq 0 }
+end
+
+# to enforce full highstate
+# once switched fully to new reclass
+#describe command('salt-call state.apply') do
+  #its('exit_status') { should eq 0 }
+  ##its('stdout') { should match (/Failed: 0/) }
+#end
diff --git a/test/integration/cluster/02_verify_minions_spec.rb b/test/integration/cluster/02_verify_minions_spec.rb
new file mode 100644
index 0000000..63becc0
--- /dev/null
+++ b/test/integration/cluster/02_verify_minions_spec.rb
@@ -0,0 +1,6 @@
+
+describe command('bash /srv/salt/bootstrap.sh') do
+  its('exit_status') { should eq 0 }
+  #its('stdout') { should match (/XCVDSGSG/) }
+end
+
diff --git a/verify.sh b/verify.sh
deleted file mode 100755
index 3b94176..0000000
--- a/verify.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-#export HOSTNAME=${`hostname -s`}
-#export DOMAIN=${`hostname -d`}
-cd /srv/salt/scripts; git pull -r || true; source bootstrap.sh || exit 1
-
-# BOOTSTRAP
-if [[ $BOOTSTRAP =~ ^(True|true|1|yes)$ ]]; then
-  # workarounds for kitchen
-  test ! -e /tmp/kitchen  || (mkdir -p /srv/salt/reclass; rsync -avh /tmp/kitchen/ /srv/salt/reclass)
-  cd /srv/salt/reclass
-  # clone latest system-level if missing
-  if [[ -e .gitmodules ]] && [[ ! -e classes/system/linux ]]; then
-    git submodule update --init --recursive --remote || true
-  fi
-  source_local_envs
-  /srv/salt/scripts/bootstrap.sh &&\
-  if [[ -e /tmp/kitchen ]]; then sed -i '/BOOTSTRAP=/d' /kitchen.env; fi
-fi
-
-# VERIFY
-export RECLASS_IGNORE_CLASS_NOTFOUND=False
-cd /srv/salt/reclass &&\
-if [[ -z "$1" ]] ; then
-  verify_salt_master &&\
-  verify_salt_minions
-else
-  verify_salt_minion "$1"
-fi