Merge "[CVP] Add method to fetch v3 or v2 keystone credentials"
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index e0e784a..e125f50 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -8,11 +8,13 @@
 
 /**
  * Run docker container with basic (keystone) parameters
+ * For backward compatibility. Deprecated.
+ * Will be removed soon.
  *
  * @param target            Host to run container
  * @param dockerImageLink   Docker image link. May be custom or default rally image
  */
-def runBasicContainer(master, target, dockerImageLink="xrally/xrally-openstack:0.9.1"){
+def runBasicContainer(master, target, dockerImageLink="xrally/xrally-openstack:0.10.1"){
     def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
     def _pillar = salt.getPillar(master, 'I@keystone:server', 'keystone:server')
@@ -27,6 +29,37 @@
             "-e OS_REGION_NAME=${keystone.region} -e OS_ENDPOINT_TYPE=admin --entrypoint /bin/bash ${dockerImageLink}")
 }
 
+
+/**
+ * Run docker container with parameters
+ *
+ * @param target            Host to run container
+ * @param dockerImageLink   Docker image link. May be custom or default rally image
+ * @param name              Name for container
+ * @param env_var           Environment variables to set in container
+ * @param entrypoint        Set entrypoint to /bin/bash or leave default
+**/
+
+
+def runContainer(master, target, dockerImageLink, name='cvp', env_var=[], entrypoint=true){
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def variables = ''
+    def entry_point = ''
+    if ( salt.cmdRun(master, target, "docker ps -f name=${name} -q", false, null, false)['return'][0].values()[0] ) {
+        salt.cmdRun(master, target, "docker rm -f ${name}")
+    }
+    if (env_var.size() > 0) {
+        variables = ' -e ' + env_var.join(' -e ')
+    }
+    if (entrypoint) {
+        entry_point = '--entrypoint /bin/bash'
+    }
+    salt.cmdRun(master, target, "docker run -tid --net=host --name=${name} " +
+                                "-u root ${entry_point} ${variables} ${dockerImageLink}")
+}
+
+
 /**
  * Get v2 Keystone credentials from pillars
  *
@@ -185,6 +218,7 @@
 
 /**
  * Execute mcp sanity tests
+ * Deprecated. Will be removed soon
  *
  * @param salt_url          Salt master url
  * @param salt_credentials  Salt credentials
@@ -219,6 +253,47 @@
  * @param env_vars          Additional environment variables for cvp-sanity-checks
  * @param output_dir        Directory for results
  */
+def runPyTests(salt_url, salt_credentials, test_set="", env_vars="", name='cvp', container_node="", remote_dir='/root/qa_results/', artifacts_dir='validation_artifacts/') {
+    def xml_file = "${name}_report.xml"
+    def common = new com.mirantis.mk.Common()
+    def salt = new com.mirantis.mk.Salt()
+    def creds = common.getCredentials(salt_credentials)
+    def username = creds.username
+    def password = creds.password
+    if (container_node != "") {
+        def saltMaster
+        saltMaster = salt.connection(salt_url, salt_credentials)
+        def script = "pytest --junitxml ${xml_file} --tb=short -sv ${test_set}"
+        env_vars.addAll("SALT_USERNAME=${username}", "SALT_PASSWORD=${password}",
+                        "SALT_URL=${salt_url}")
+        variables = ' -e ' + env_vars.join(' -e ')
+        salt.cmdRun(saltMaster, container_node, "docker exec ${variables} ${name} bash -c '${script}'", false)
+        salt.cmdRun(saltMaster, container_node, "docker cp ${name}:/var/lib/${xml_file} ${remote_dir}${xml_file}")
+        addFiles(saltMaster, container_node, remote_dir+xml_file, artifacts_dir)
+    }
+    else {
+        if (env_vars.size() > 0) {
+        variables = 'export ' + env_vars.join(';export ')
+        }
+        def script = ". ${env.WORKSPACE}/venv/bin/activate; ${variables}; " +
+                     "pytest --junitxml ${artifacts_dir}${xml_file} --tb=short -sv ${env.WORKSPACE}/${test_set}"
+        withEnv(["SALT_USERNAME=${username}", "SALT_PASSWORD=${password}", "SALT_URL=${salt_url}"]) {
+            def statusCode = sh script:script, returnStatus:true
+        }
+    }
+}
+
+/**
+ * Execute pytest framework tests
+ * For backward compatibility
+ * Will be removed soon
+ *
+ * @param salt_url          Salt master url
+ * @param salt_credentials  Salt credentials
+ * @param test_set          Test set to run
+ * @param env_vars          Additional environment variables for cvp-sanity-checks
+ * @param output_dir        Directory for results
+ */
 def runTests(salt_url, salt_credentials, test_set="", output_dir="validation_artifacts/", env_vars="") {
     def common = new com.mirantis.mk.Common()
     def creds = common.getCredentials(salt_credentials)
@@ -533,12 +608,12 @@
  * @param testing_tools_repo    	Repo with testing tools: configuration script, skip-list, etc.
  * @param tempest_repo         		Tempest repo to clone. Can be upstream tempest (default, recommended), your customized tempest in local/remote repo or path inside container. If not specified, tempest will not be configured.
  * @param tempest_endpoint_type         internalURL or adminURL or publicURL to use in tests
- * @param tempest_version	        Version of tempest to use
+ * @param tempest_version	        Version of tempest to use. This value will be just passed to configure.sh script (cvp-configuration repo).
  * @param conf_script_path              Path to configuration script.
  * @param ext_variables                 Some custom extra variables to add into container
  */
 def configureContainer(master, target, proxy, testing_tools_repo, tempest_repo,
-                       tempest_endpoint_type="internalURL", tempest_version="15.0.0",
+                       tempest_endpoint_type="internalURL", tempest_version="",
                        conf_script_path="", ext_variables = []) {
     def salt = new com.mirantis.mk.Salt()
     if (testing_tools_repo != "" ) {
@@ -571,19 +646,17 @@
     def salt = new com.mirantis.mk.Salt()
     def xml_file = "${output_filename}.xml"
     def html_file = "${output_filename}.html"
-    def log_file = "${output_filename}.log"
     skip_list_cmd = ''
     if (skip_list != '') {
         skip_list_cmd = "--skip-list ${skip_list}"
     }
-    salt.cmdRun(master, target, "docker exec cvp rally verify start --pattern ${test_pattern} ${skip_list_cmd} " +
-                                "--detailed > ${log_file}", false)
-    salt.cmdRun(master, target, "cat ${log_file}")
+    salt.cmdRun(master, target, "docker exec cvp rally verify start --pattern ${test_pattern} ${skip_list_cmd} --detailed")
     salt.cmdRun(master, target, "docker exec cvp rally verify report --type junit-xml --to /home/rally/${xml_file}")
     salt.cmdRun(master, target, "docker exec cvp rally verify report --type html --to /home/rally/${html_file}")
     salt.cmdRun(master, target, "docker cp cvp:/home/rally/${xml_file} ${output_dir}")
     salt.cmdRun(master, target, "docker cp cvp:/home/rally/${html_file} ${output_dir}")
-    return salt.cmdRun(master, target, "docker exec cvp rally verify show | head -5 | tail -1 | awk '{print \$4}'")['return'][0].values()[0].split()[0]
+    return salt.cmdRun(master, target, "docker exec cvp rally verify show | head -5 | tail -1 | " +
+                                       "awk '{print \$4}'")['return'][0].values()[0].split()[0]
 }
 
 /**
@@ -597,10 +670,8 @@
 def runCVPrally(master, target, scenarios_path, output_dir, output_filename="docker-rally") {
     def salt = new com.mirantis.mk.Salt()
     def xml_file = "${output_filename}.xml"
-    def log_file = "${output_filename}.log"
     def html_file = "${output_filename}.html"
-    salt.cmdRun(master, target, "docker exec cvp rally task start ${scenarios_path} > ${log_file}", false)
-    salt.cmdRun(master, target, "cat ${log_file}")
+    salt.cmdRun(master, target, "docker exec cvp rally task start ${scenarios_path}")
     salt.cmdRun(master, target, "docker exec cvp rally task report --out ${html_file}")
     salt.cmdRun(master, target, "docker exec cvp rally task report --junit --out ${xml_file}")
     salt.cmdRun(master, target, "docker cp cvp:/home/rally/${xml_file} ${output_dir}")
@@ -699,7 +770,7 @@
  */
 def get_vip_node(master, target) {
     def salt = new com.mirantis.mk.Salt()
-    def list = salt.runSaltProcessStep(master, "${target}", 'cmd.run', ["ip a | grep global | grep -v brd"])['return'][0]
+    def list = salt.runSaltProcessStep(master, "${target}", 'cmd.run', ["ip a | grep '/32'"])['return'][0]
     for (item in list.keySet()) {
         if (list[item]) {
             return item
@@ -722,14 +793,12 @@
  * Cleanup
  *
  * @param target            Host to run commands
+ * @param name              Name of container to remove
  */
-def runCleanup(master, target) {
+def runCleanup(master, target, name='cvp') {
     def salt = new com.mirantis.mk.Salt()
-    if ( salt.cmdRun(master, target, "docker ps -f name=qa_tools -q", false, null, false)['return'][0].values()[0] ) {
-        salt.cmdRun(master, target, "docker rm -f qa_tools")
-    }
-    if ( salt.cmdRun(master, target, "docker ps -f name=cvp -q", false, null, false)['return'][0].values()[0] ) {
-        salt.cmdRun(master, target, "docker rm -f cvp")
+    if ( salt.cmdRun(master, target, "docker ps -f name=${name} -q", false, null, false)['return'][0].values()[0] ) {
+        salt.cmdRun(master, target, "docker rm -f ${name}")
     }
 }
 /**
diff --git a/src/com/mirantis/mk/Http.groovy b/src/com/mirantis/mk/Http.groovy
index 987a998..b752b42 100644
--- a/src/com/mirantis/mk/Http.groovy
+++ b/src/com/mirantis/mk/Http.groovy
@@ -112,25 +112,25 @@
 }
 
 /**
- * Make generic call using Salt REST API and return parsed JSON
+ * Make generic call using REST API and return parsed JSON
  *
- * @param master   Salt connection object
- * @param uri   URI which will be appended to Salt server base URL
- * @param method    HTTP method to use (default GET)
- * @param data      JSON data to POST or PUT
- * @param headers   Map of additional request headers
+ * @param base    connection object, map with 'url' and optional 'authToken' keys
+ * @param uri     URI which will be appended to connection base URL
+ * @param method  HTTP method to use (default GET)
+ * @param data    JSON data to POST, PUT or PATCH
+ * @param headers Map of additional request headers
  */
-def restCall(master, uri, method = 'GET', data = null, headers = [:]) {
-    def connection = new URL("${master.url}${uri}").openConnection()
+def restCall(base, uri, method = 'GET', data = null, headers = [:]) {
+    def connection = new URL("${base.url}${uri}").openConnection()
     if (method != 'GET') {
         connection.setRequestMethod(method)
     }
 
     connection.setRequestProperty('User-Agent', 'jenkins-groovy')
     connection.setRequestProperty('Accept', 'application/json')
-    if (master.authToken) {
-        // XXX: removeme
-        connection.setRequestProperty('X-Auth-Token', master.authToken)
+    if (base.authToken) {
+        // XXX: removeme, explicitly use headers instead
+        connection.setRequestProperty('X-Auth-Token', base.authToken)
     }
 
     for (header in headers) {
@@ -163,34 +163,56 @@
 }
 
 /**
- * Make GET request using Salt REST API and return parsed JSON
+ * Make GET request using REST API and return parsed JSON
  *
- * @param master   Salt connection object
- * @param uri   URI which will be appended to Salt server base URL
+ * @param base  connection object, map with 'url' and optional 'authToken' keys
+ * @param uri   URI which will be appended to server base URL
  */
-def restGet(master, uri, data = null) {
-    return restCall(master, uri, 'GET', data)
+def restGet(base, uri, data = null, headers = [:]) {
+    return restCall(base, uri, 'GET', data, headers)
 }
 
 /**
- * Make POST request using Salt REST API and return parsed JSON
+ * Make POST request using REST API and return parsed JSON
  *
- * @param master   Salt connection object
- * @param uri   URI which will be appended to Docker server base URL
+ * @param base  connection object, map with 'url' and optional 'authToken' keys
+ * @param uri   URI which will be appended to server base URL
+ * @param data  JSON Data to POST
+ */
+def restPost(base, uri, data = null, headers = ['Accept': '*/*']) {
+    return restCall(base, uri, 'POST', data, headers)
+}
+
+/**
+ * Make PUT request using REST API and return parsed JSON
+ *
+ * @param base  connection object, map with 'url' and optional 'authToken' keys
+ * @param uri   URI which will be appended to server base URL
  * @param data  JSON Data to PUT
  */
-def restPost(master, uri, data = null) {
-    return restCall(master, uri, 'POST', data, ['Accept': '*/*'])
+def restPut(base, uri, data = null, headers = ['Accept': '*/*']) {
+    return restCall(base, uri, 'PUT', data, headers)
 }
 
 /**
- * Make DELETE request using Salt REST API and return parsed JSON
+ * Make PATCH request using REST API and return parsed JSON
  *
- * @param master   Salt connection object
- * @param uri   URI which will be appended to Salt server base URL
+ * @param base  connection object, map with 'url' and optional 'authToken' keys
+ * @param uri   URI which will be appended to server base URL
+ * @param data  JSON Data to PUT
  */
-def restDelete(master, uri, data = null) {
-    return restCall(master, uri, 'DELETE', data)
+def restPatch(base, uri, data = null, headers = ['Accept': '*/*']) {
+    return restCall(base, uri, 'PATCH', data, headers)
+}
+
+/**
+ * Make DELETE request using REST API and return parsed JSON
+ *
+ * @param base  connection object, map with 'url' and optional 'authToken' keys
+ * @param uri   URI which will be appended to server base URL
+ */
+def restDelete(base, uri, data = null, headers = [:]) {
+    return restCall(base, uri, 'DELETE', data, headers)
 }
 
 /**
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 265cfa0..f56e28b 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -544,7 +544,7 @@
     salt.enforceStateWithExclude(master, "I@opencontrail:collector ${extra_tgt}", "opencontrail", "opencontrail.client")
 
     salt.enforceStateWithTest(master, "( I@opencontrail:control or I@opencontrail:collector ) ${extra_tgt}", 'docker.client', "I@docker:client and I@opencontrail:control ${extra_tgt}")
-    installBackup(master, 'contrail', extra_tgt)
+    // NOTE(ivasilevskaya) call to installBackup here has been removed as it breaks deployment if done before computes are deployed
 }
 
 
@@ -1267,4 +1267,4 @@
     else {
       common.infoMsg("No applications found for orchestration")
     }
-}
\ No newline at end of file
+}
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 2d1a888..e46a733 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -94,7 +94,7 @@
                         common.debianExtraRepos(extraReposYaml)
                         sh('''#!/bin/bash -xe
                             apt-get update
-                            apt-get install -y python-netaddr reclass
+                            apt-get install -y python-netaddr
                         ''')
 
                     }
@@ -270,6 +270,10 @@
             ''')
         },
 
+        '003_Install_Reclass_package'    : {
+            sh('apt-get install -y reclass')
+        },
+
         '004_Run_tests'                  : {
             def testTimeout = 40 * 60
             timeout(time: testTimeout, unit: 'SECONDS') {