Merge "Add retry for salt.minion state"
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index 9e205cb..102ec2b 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -8,11 +8,13 @@
 
 /**
  * Run docker container with basic (keystone) parameters
+ * For backward compatibility. Deprecated.
+ * Will be removed soon.
  *
  * @param target            Host to run container
  * @param dockerImageLink   Docker image link. May be custom or default rally image
  */
-def runBasicContainer(master, target, dockerImageLink="xrally/xrally-openstack:0.9.1"){
+def runBasicContainer(master, target, dockerImageLink="xrally/xrally-openstack:0.10.1"){
     def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
     def _pillar = salt.getPillar(master, 'I@keystone:server', 'keystone:server')
@@ -27,6 +29,86 @@
             "-e OS_REGION_NAME=${keystone.region} -e OS_ENDPOINT_TYPE=admin --entrypoint /bin/bash ${dockerImageLink}")
 }
 
+
+/**
+ * Run docker container with parameters
+ *
+ * @param target            Host to run container
+ * @param dockerImageLink   Docker image link. May be custom or default rally image
+ * @param name              Name for container
+ * @param env_var           Environment variables to set in container
+ * @param entrypoint        Set entrypoint to /bin/bash or leave default
+**/
+
+
+def runContainer(master, target, dockerImageLink, name='cvp', env_var=[], entrypoint=true){
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def variables = ''
+    def entry_point = ''
+    if ( salt.cmdRun(master, target, "docker ps -f name=${name} -q", false, null, false)['return'][0].values()[0] ) {
+        salt.cmdRun(master, target, "docker rm -f ${name}")
+    }
+    if (env_var.size() > 0) {
+        variables = ' -e ' + env_var.join(' -e ')
+    }
+    if (entrypoint) {
+        entry_point = '--entrypoint /bin/bash'
+    }
+    salt.cmdRun(master, target, "docker run -tid --net=host --name=${name} " +
+                                "-u root ${entry_point} ${variables} ${dockerImageLink}")
+}
+
+
+/**
+ * Get v2 Keystone credentials from pillars
+ *
+ */
+def _get_keystone_creds_v2(master){
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def keystone = []
+    common.infoMsg("Fetching Keystone v2 credentials")
+    _pillar = salt.getPillar(master, 'I@keystone:server', 'keystone:server')['return'][0].values()[0]
+    keystone.add("OS_USERNAME=${_pillar.admin_name}")
+    keystone.add("OS_PASSWORD=${_pillar.admin_password}")
+    keystone.add("OS_TENANT_NAME=${_pillar.admin_tenant}")
+    keystone.add("OS_AUTH_URL=http://${_pillar.bind.private_address}:${_pillar.bind.private_port}/v2.0")
+    keystone.add("OS_REGION_NAME=${_pillar.region}")
+    keystone.add("OS_ENDPOINT_TYPE=admin")
+    return keystone
+}
+
+/**
+ * Get v3 Keystone credentials from pillars
+ *
+ */
+def _get_keystone_creds_v3(master){
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    pillar_name = 'keystone:client:os_client_config:cfgs:root:content:clouds:admin_identity'
+    common.infoMsg("Fetching Keystone v3 credentials")
+    def _pillar = salt.getPillar(master, 'I@keystone:client', pillar_name)['return'][0].values()[0]
+    def keystone = []
+    if (_pillar) {
+        keystone.add("OS_USERNAME=${_pillar.auth.username}")
+        keystone.add("OS_PASSWORD=${_pillar.auth.password}")
+        keystone.add("OS_TENANT_NAME=${_pillar.auth.project_name}")
+        keystone.add("OS_PROJECT_NAME=${_pillar.auth.project_name}")
+        keystone.add("OS_AUTH_URL=${_pillar.auth.auth_url}/v3")
+        keystone.add("OS_REGION_NAME=${_pillar.region_name}")
+        keystone.add("OS_IDENTITY_API_VERSION=${_pillar.identity_api_version}")
+        keystone.add("OS_ENDPOINT_TYPE=admin")
+        keystone.add("OS_PROJECT_DOMAIN_NAME=${_pillar.auth.project_domain_name}")
+        keystone.add("OS_USER_DOMAIN_NAME=${_pillar.auth.user_domain_name}")
+        return keystone
+    }
+    else {
+        common.warningMsg("Failed to fetch Keystone v3 credentials")
+        return false
+    }
+}
+
 /**
  * Get file content (encoded). The content encoded by Base64.
  *
@@ -136,6 +218,7 @@
 
 /**
  * Execute mcp sanity tests
+ * Deprecated. Will be removed soon
  *
  * @param salt_url          Salt master url
  * @param salt_credentials  Salt credentials
@@ -170,6 +253,47 @@
  * @param env_vars          Additional environment variables for cvp-sanity-checks
  * @param output_dir        Directory for results
  */
+def runPyTests(salt_url, salt_credentials, test_set="", env_vars="", name='cvp', container_node="", remote_dir='/root/qa_results/', artifacts_dir='validation_artifacts/') {
+    def xml_file = "${name}_report.xml"
+    def common = new com.mirantis.mk.Common()
+    def salt = new com.mirantis.mk.Salt()
+    def creds = common.getCredentials(salt_credentials)
+    def username = creds.username
+    def password = creds.password
+    if (container_node != "") {
+        def saltMaster
+        saltMaster = salt.connection(salt_url, salt_credentials)
+        def script = "pytest --junitxml ${xml_file} --tb=short -sv ${test_set}"
+        env_vars.addAll("SALT_USERNAME=${username}", "SALT_PASSWORD=${password}",
+                        "SALT_URL=${salt_url}")
+        variables = ' -e ' + env_vars.join(' -e ')
+        salt.cmdRun(saltMaster, container_node, "docker exec ${variables} ${name} bash -c '${script}'", false)
+        salt.cmdRun(saltMaster, container_node, "docker cp ${name}:/var/lib/${xml_file} ${remote_dir}${xml_file}")
+        addFiles(saltMaster, container_node, remote_dir+xml_file, artifacts_dir)
+    }
+    else {
+        if (env_vars.size() > 0) {
+        variables = 'export ' + env_vars.join(';export ')
+        }
+        def script = ". ${env.WORKSPACE}/venv/bin/activate; ${variables}; " +
+                     "pytest --junitxml ${artifacts_dir}${xml_file} --tb=short -sv ${env.WORKSPACE}/${test_set}"
+        withEnv(["SALT_USERNAME=${username}", "SALT_PASSWORD=${password}", "SALT_URL=${salt_url}"]) {
+            def statusCode = sh script:script, returnStatus:true
+        }
+    }
+}
+
+/**
+ * Execute pytest framework tests
+ * For backward compatibility
+ * Will be removed soon
+ *
+ * @param salt_url          Salt master url
+ * @param salt_credentials  Salt credentials
+ * @param test_set          Test set to run
+ * @param env_vars          Additional environment variables for cvp-sanity-checks
+ * @param output_dir        Directory for results
+ */
 def runTests(salt_url, salt_credentials, test_set="", output_dir="validation_artifacts/", env_vars="") {
     def common = new com.mirantis.mk.Common()
     def creds = common.getCredentials(salt_credentials)
@@ -328,7 +452,7 @@
       def _pillar = salt.getPillar(master, 'I@kubernetes:master and *01*', 'kubernetes:master')
       def kubernetes = _pillar['return'][0].values()[0]
       env_vars = [
-          "KUBERNETES_HOST=${kubernetes.apiserver.vip_address}" +
+          "KUBERNETES_HOST=http://${kubernetes.apiserver.vip_address}" +
           ":${kubernetes.apiserver.insecure_port}",
           "KUBERNETES_CERT_AUTH=${dest_folder}/k8s-ca.crt",
           "KUBERNETES_CLIENT_KEY=${dest_folder}/k8s-client.key",
@@ -370,8 +494,11 @@
         break
       }
     }
-    cmd_rally_report= "rally task export --type junit-xml --to ${dest_folder}/report-rally.xml; " +
-        "rally task report --out ${dest_folder}/report-rally.html"
+    cmd_rally_report= "rally task export --uuid \\\$(rally task list --uuids-only --status finished) " +
+        "--type junit-xml --to ${dest_folder}/report-rally.xml; " +
+        "rally task report --uuid \\\$(rally task list --uuids-only --status finished) " +
+        "--out ${dest_folder}/report-rally.html"
+
     full_cmd = 'set -xe; ' + cmd_rally_plugins +
         cmd_rally_init + cmd_rally_checkout +
         'set +e; ' + cmd_rally_start +
@@ -484,12 +611,12 @@
  * @param testing_tools_repo    	Repo with testing tools: configuration script, skip-list, etc.
  * @param tempest_repo         		Tempest repo to clone. Can be upstream tempest (default, recommended), your customized tempest in local/remote repo or path inside container. If not specified, tempest will not be configured.
  * @param tempest_endpoint_type         internalURL or adminURL or publicURL to use in tests
- * @param tempest_version	        Version of tempest to use
+ * @param tempest_version	        Version of tempest to use. This value will be just passed to configure.sh script (cvp-configuration repo).
  * @param conf_script_path              Path to configuration script.
  * @param ext_variables                 Some custom extra variables to add into container
  */
 def configureContainer(master, target, proxy, testing_tools_repo, tempest_repo,
-                       tempest_endpoint_type="internalURL", tempest_version="15.0.0",
+                       tempest_endpoint_type="internalURL", tempest_version="",
                        conf_script_path="", ext_variables = []) {
     def salt = new com.mirantis.mk.Salt()
     if (testing_tools_repo != "" ) {
@@ -522,19 +649,17 @@
     def salt = new com.mirantis.mk.Salt()
     def xml_file = "${output_filename}.xml"
     def html_file = "${output_filename}.html"
-    def log_file = "${output_filename}.log"
     skip_list_cmd = ''
     if (skip_list != '') {
         skip_list_cmd = "--skip-list ${skip_list}"
     }
-    salt.cmdRun(master, target, "docker exec cvp rally verify start --pattern ${test_pattern} ${skip_list_cmd} " +
-                                "--detailed > ${log_file}", false)
-    salt.cmdRun(master, target, "cat ${log_file}")
+    salt.cmdRun(master, target, "docker exec cvp rally verify start --pattern ${test_pattern} ${skip_list_cmd} --detailed")
     salt.cmdRun(master, target, "docker exec cvp rally verify report --type junit-xml --to /home/rally/${xml_file}")
     salt.cmdRun(master, target, "docker exec cvp rally verify report --type html --to /home/rally/${html_file}")
     salt.cmdRun(master, target, "docker cp cvp:/home/rally/${xml_file} ${output_dir}")
     salt.cmdRun(master, target, "docker cp cvp:/home/rally/${html_file} ${output_dir}")
-    return salt.cmdRun(master, target, "docker exec cvp rally verify show | head -5 | tail -1 | awk '{print \$4}'")['return'][0].values()[0].split()[0]
+    return salt.cmdRun(master, target, "docker exec cvp rally verify show | head -5 | tail -1 | " +
+                                       "awk '{print \$4}'")['return'][0].values()[0].split()[0]
 }
 
 /**
@@ -548,10 +673,8 @@
 def runCVPrally(master, target, scenarios_path, output_dir, output_filename="docker-rally") {
     def salt = new com.mirantis.mk.Salt()
     def xml_file = "${output_filename}.xml"
-    def log_file = "${output_filename}.log"
     def html_file = "${output_filename}.html"
-    salt.cmdRun(master, target, "docker exec cvp rally task start ${scenarios_path} > ${log_file}", false)
-    salt.cmdRun(master, target, "cat ${log_file}")
+    salt.cmdRun(master, target, "docker exec cvp rally task start ${scenarios_path}")
     salt.cmdRun(master, target, "docker exec cvp rally task report --out ${html_file}")
     salt.cmdRun(master, target, "docker exec cvp rally task report --junit --out ${xml_file}")
     salt.cmdRun(master, target, "docker cp cvp:/home/rally/${xml_file} ${output_dir}")
@@ -650,7 +773,7 @@
  */
 def get_vip_node(master, target) {
     def salt = new com.mirantis.mk.Salt()
-    def list = salt.runSaltProcessStep(master, "${target}", 'cmd.run', ["ip a | grep global | grep -v brd"])['return'][0]
+    def list = salt.runSaltProcessStep(master, "${target}", 'cmd.run', ["ip a | grep '/32'"])['return'][0]
     for (item in list.keySet()) {
         if (list[item]) {
             return item
@@ -673,14 +796,12 @@
  * Cleanup
  *
  * @param target            Host to run commands
+ * @param name              Name of container to remove
  */
-def runCleanup(master, target) {
+def runCleanup(master, target, name='cvp') {
     def salt = new com.mirantis.mk.Salt()
-    if ( salt.cmdRun(master, target, "docker ps -f name=qa_tools -q", false, null, false)['return'][0].values()[0] ) {
-        salt.cmdRun(master, target, "docker rm -f qa_tools")
-    }
-    if ( salt.cmdRun(master, target, "docker ps -f name=cvp -q", false, null, false)['return'][0].values()[0] ) {
-        salt.cmdRun(master, target, "docker rm -f cvp")
+    if ( salt.cmdRun(master, target, "docker ps -f name=${name} -q", false, null, false)['return'][0].values()[0] ) {
+        salt.cmdRun(master, target, "docker rm -f ${name}")
     }
 }
 /**
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index 87b1696..09f1b51 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -443,6 +443,37 @@
 }
 
 /**
+ *
+ * Deep merge of  Map items. Merges variable number of maps in to onto.
+ *   Using the following rules:
+ *     - Lists are appended
+ *     - Maps are updated
+ *     - other object types are replaced.
+ *
+ *
+ * @param onto Map object to merge in
+ * @param overrides Map objects to merge to onto
+*/
+def mergeMaps(Map onto, Map... overrides){
+    if (!overrides){
+        return onto
+    }
+    else if (overrides.length == 1) {
+        overrides[0]?.each { k, v ->
+            if (v in Map && onto[k] in Map){
+                mergeMaps((Map) onto[k], (Map) v)
+            } else if (v in List) {
+                onto[k] += v
+            } else {
+                onto[k] = v
+            }
+        }
+        return onto
+    }
+    return overrides.inject(onto, { acc, override -> mergeMaps(acc, override ?: [:]) })
+}
+
+/**
  * Test pipeline input parameter existence and validity (not null and not empty string)
  * @param paramName input parameter name (usually uppercase)
   */
@@ -888,33 +919,58 @@
 
 /**
  * Ugly processing basic funcs with /etc/apt
- * @param configYaml
+ * @param repoConfig YAML text or Map
  * Example :
- configYaml = '''
+ repoConfig = '''
  ---
- distrib_revision: 'nightly'
  aprConfD: |-
-    APT::Get::AllowUnauthenticated 'true';
+   APT::Get::AllowUnauthenticated 'true';
  repo:
-    mcp_saltstack:
-        source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/saltstack-2017.7/xenial xenial main"
-        pinning: |-
-            Package: libsodium18
-            Pin: release o=SaltStack
-            Pin-Priority: 50
+   mcp_saltstack:
+     source: "deb [arch=amd64] http://mirror.mirantis.com/nightly/saltstack-2017.7/xenial xenial main"
+     pin:
+       - package: "libsodium18"
+         pin: "release o=SaltStack"
+         priority: 50
+       - package: "*"
+         pin: "release o=SaltStack"
+         priority: "1100"
+     repo_key: "http://mirror.mirantis.com/public.gpg"
  '''
  *
  */
 
-def debianExtraRepos(configYaml) {
-    def config = readYaml text: configYaml
-    def distribRevision = config.get('distrib_revision', 'nightly')
+def debianExtraRepos(repoConfig) {
+    def config = null
+    if (repoConfig instanceof Map) {
+        config = repoConfig
+    } else {
+        config = readYaml text: repoConfig
+    }
     if (config.get('repo', false)) {
         for (String repo in config['repo'].keySet()) {
-            source = config['repo'][repo]['source'].replace('SUB_DISTRIB_REVISION', distribRevision)
+            source = config['repo'][repo]['source']
             warningMsg("Write ${source} >  /etc/apt/sources.list.d/${repo}.list")
             sh("echo '${source}' > /etc/apt/sources.list.d/${repo}.list")
-            // TODO implement pining
+            if (config['repo'][repo].containsKey('repo_key')) {
+                key = config['repo'][repo]['repo_key']
+                sh("wget -O - '${key}' | apt-key add -")
+            }
+            if (config['repo'][repo]['pin']) {
+                def repoPins = []
+                for (Map pin in config['repo'][repo]['pin']) {
+                    repoPins.add("Package: ${pin['package']}")
+                    repoPins.add("Pin: ${pin['pin']}")
+                    repoPins.add("Pin-Priority: ${pin['priority']}")
+                }
+                if (repoPins) {
+                    repoPins.add(0, "### Extra ${repo} repo pin start ###")
+                    repoPins.add("### Extra ${repo} repo pin end ###")
+                    repoPinning = repoPins.join('\n')
+                    warningMsg("Adding pinning \n${repoPinning}\n => /etc/apt/preferences.d/${repo}")
+                    sh("echo '${repoPinning}' > /etc/apt/preferences.d/${repo}")
+                }
+            }
         }
     }
     if (config.get('aprConfD', false)) {
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 5c762ae..2c6e870 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -36,6 +36,8 @@
     salt.enforceState(master, "I@salt:master ${extra_tgt}", ['salt.minion'])
     salt.fullRefresh(master, "* ${extra_tgt}")
     salt.enforceState(master, "* ${extra_tgt}", ['linux.network.proxy'], true, false, null, false, 60, 2)
+    // Make sure all repositories are in place before proceeding with package installation from other states
+    salt.enforceState(master, "* ${extra_tgt}", ['linux.system.repo'], true, false, null, false, 60, 2)
     try {
         salt.enforceState(master, "* ${extra_tgt}", ['salt.minion.base'], true, false, null, false, 60, 2)
         sleep(5)
@@ -553,7 +555,7 @@
     salt.enforceStateWithExclude(master, "I@opencontrail:collector ${extra_tgt}", "opencontrail", "opencontrail.client")
 
     salt.enforceStateWithTest(master, "( I@opencontrail:control or I@opencontrail:collector ) ${extra_tgt}", 'docker.client', "I@docker:client and I@opencontrail:control ${extra_tgt}")
-    installBackup(master, 'contrail', extra_tgt)
+    // NOTE(ivasilevskaya) call to installBackup here has been removed as it breaks deployment if done before computes are deployed
 }
 
 
@@ -1276,4 +1278,4 @@
     else {
       common.infoMsg("No applications found for orchestration")
     }
-}
\ No newline at end of file
+}
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index eb4b19c..f135cc0 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -270,6 +270,89 @@
 }
 
 /**
+ *
+ * @param context - context template
+ * @param contextName - context template name
+ * @param saltMasterName - hostname of Salt Master node
+ * @param virtualenv - pyvenv with CC and dep's
+ * @param templateEnvDir - root of CookieCutter
+ * @return
+ */
+def generateModel(context, contextName, saltMasterName, virtualenv, modelEnv, templateEnvDir, multiModels = true) {
+    def common = new com.mirantis.mk.Common()
+    def generatedModel = multiModels ? "${modelEnv}/${contextName}" : modelEnv
+    def templateContext = readYaml text: context
+    def clusterDomain = templateContext.default_context.cluster_domain
+    def clusterName = templateContext.default_context.cluster_name
+    def outputDestination = "${generatedModel}/classes/cluster/${clusterName}"
+    def templateBaseDir = templateEnvDir
+    def templateDir = "${templateEnvDir}/dir"
+    def templateOutputDir = templateBaseDir
+    dir(templateEnvDir) {
+        common.infoMsg("Generating model from context ${contextName}")
+        def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
+        for (product in productList) {
+            // get templateOutputDir and productDir
+            templateOutputDir = "${templateEnvDir}/output/${product}"
+            productDir = product
+            templateDir = "${templateEnvDir}/cluster_product/${productDir}"
+            // Bw for 2018.8.1 and older releases
+            if (product.startsWith("stacklight") && (!fileExists(templateDir))) {
+                common.warningMsg("Old release detected! productDir => 'stacklight2' ")
+                productDir = "stacklight2"
+                templateDir = "${templateEnvDir}/cluster_product/${productDir}"
+            }
+            // generate infra unless its explicitly disabled
+            if ((product == "infra" && templateContext.default_context.get("infra_enabled", "True").toBoolean())
+                 || (templateContext.default_context.get(product + "_enabled", "False").toBoolean())) {
+
+                common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
+
+                sh "rm -rf ${templateOutputDir} || true"
+                sh "mkdir -p ${templateOutputDir}"
+                sh "mkdir -p ${outputDestination}"
+
+                buildCookiecutterTemplate(templateDir, context, templateOutputDir, virtualenv, templateBaseDir)
+                sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+            } else {
+                common.warningMsg("Product " + product + " is disabled")
+            }
+        }
+
+        def localRepositories = templateContext.default_context.local_repositories
+        localRepositories = localRepositories ? localRepositories.toBoolean() : false
+        def offlineDeployment = templateContext.default_context.offline_deployment
+        offlineDeployment = offlineDeployment ? offlineDeployment.toBoolean() : false
+        if (localRepositories && !offlineDeployment) {
+            def mcpVersion = templateContext.default_context.mcp_version
+            def aptlyModelUrl = templateContext.default_context.local_model_url
+            def ssh = new com.mirantis.mk.Ssh()
+            dir(path: modelEnv) {
+                ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
+                if (!(mcpVersion in ["nightly", "testing", "stable"])) {
+                    ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
+                }
+            }
+        }
+
+        def nodeFile = "${generatedModel}/nodes/${saltMasterName}.${clusterDomain}.yml"
+        def nodeString = """classes:
+- cluster.${clusterName}.infra.config
+parameters:
+  _param:
+    linux_system_codename: xenial
+    reclass_data_revision: master
+  linux:
+    system:
+      name: ${saltMasterName}
+      domain: ${clusterDomain}
+    """
+        sh "mkdir -p ${generatedModel}/nodes/"
+        writeFile(file: nodeFile, text: nodeString)
+    }
+}
+
+/**
  * Install jinja rendering in isolated environment
  *
  * @param path        Path where virtualenv is created
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 2d1a888..eb6e546 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -5,7 +5,7 @@
  were tests successful or not.
  * @param config - LinkedHashMap with configuration params:
  *   dockerHostname - (required) Hostname to use for Docker container.
- *   formulasRevision - (optional) Revision of packages to use (default proposed).
+ *   distribRevision - (optional) Revision of packages to use (default proposed).
  *   runCommands - (optional) Dict with closure structure of body required tests. For example:
  *     [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ]
  *     Before execution runCommands will be sorted by key names. Alpabetical order is preferred.
@@ -26,7 +26,7 @@
     // setup options
     def defaultContainerName = 'test-' + UUID.randomUUID().toString()
     def dockerHostname = config.get('dockerHostname', defaultContainerName)
-    def formulasRevision = config.get('formulasRevision', 'proposed')
+    def distribRevision = config.get('distribRevision', 'proposed')
     def runCommands = config.get('runCommands', [:])
     def runFinally = config.get('runFinally', [:])
     def baseRepoPreConfig = config.get('baseRepoPreConfig', true)
@@ -35,7 +35,7 @@
     def dockerMaxCpus = config.get('dockerMaxCpus', 4)
     def dockerExtraOpts = config.get('dockerExtraOpts', [])
     def envOpts = config.get('envOpts', [])
-    envOpts.add("DISTRIB_REVISION=${formulasRevision}")
+    envOpts.add("DISTRIB_REVISION=${distribRevision}")
     def dockerBaseOpts = [
         '-u root:root',
         "--hostname=${dockerHostname}",
@@ -43,37 +43,64 @@
         "--name=${dockerContainerName}",
         "--cpus=${dockerMaxCpus}"
     ]
-
     def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
-    def defaultExtraReposYaml = '''
+    def extraReposConfig = null
+    if (baseRepoPreConfig) {
+        // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release
+        def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main"
+        try {
+            def releaseNaming = 'yyyy.MM.dd'
+            def repoDateUsed = new Date().parse(releaseNaming, distribRevision)
+            def extraAvailableFrom = new Date().parse(releaseNaming, '2018.11.0')
+            if (repoDateUsed < extraAvailableFrom) {
+              extraRepoSource = "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
+            }
+        } catch (Exception e) {
+            common.warningMsg(e)
+            if ( !(distribRevision in [ 'nightly', 'proposed', 'testing' ] )) {
+                extraRepoSource = "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
+            }
+        }
+
+        def defaultExtraReposYaml = """
 ---
-distrib_revision: 'nightly'
 aprConfD: |-
   APT::Get::AllowUnauthenticated 'true';
   APT::Get::Install-Suggests 'false';
   APT::Get::Install-Recommends 'false';
 repo:
   mcp_saltstack:
-    source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/saltstack-2017.7/xenial xenial main"
-    pinning: |-
-        Package: libsodium18
-        Pin: release o=SaltStack
-        Pin-Priority: 50
-
-        Package: *
-        Pin: release o=SaltStack
-        Pin-Priority: 1100
+    source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/saltstack-2017.7/xenial xenial main"
+    pin:
+      - package: "libsodium18"
+        pin: "release o=SaltStack"
+        priority: 50
+      - package: "*"
+        pin: "release o=SaltStack"
+        priority: "1100"
   mcp_extra:
-    source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/extra/xenial xenial main"
+    source: "${extraRepoSource}"
+  mcp_saltformulas:
+    source: "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} salt salt-latest"
+    repo_key: "http://apt.mcp.mirantis.net/public.gpg"
   ubuntu:
-    source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial main restricted universe"
+    source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe"
   ubuntu-upd:
-    source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial-updates main restricted universe"
+    source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-updates main restricted universe"
   ubuntu-sec:
-    source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial-security main restricted universe"
-'''
+    source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-security main restricted universe"
+"""
+        // override for now
+        def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'override')
+        def extraRepos = config.get('extraRepos', [:])
+        def defaultRepos = readYaml text: defaultExtraReposYaml
+        if (extraRepoMergeStrategy == 'merge') {
+            extraReposConfig = common.mergeMaps(defaultRepos, extraRepos)
+        } else {
+            extraReposConfig = extraRepos ? extraRepos : defaultRepos
+        }
+    }
     def img = docker.image(dockerImageName)
-    def extraReposYaml = config.get('extraReposYaml', defaultExtraReposYaml)
 
     img.pull()
 
@@ -90,11 +117,12 @@
                             echo "Installing extra-deb dependencies inside docker:"
                             echo > /etc/apt/sources.list
                             rm -vf /etc/apt/sources.list.d/* || true
+                            rm -vf /etc/apt/preferences.d/* || true
                         """)
-                        common.debianExtraRepos(extraReposYaml)
+                        common.debianExtraRepos(extraReposConfig)
                         sh('''#!/bin/bash -xe
                             apt-get update
-                            apt-get install -y python-netaddr reclass
+                            apt-get install -y python-netaddr
                         ''')
 
                     }
@@ -164,7 +192,7 @@
     sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
     sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
     def configRun = [
-        'formulasRevision': distribRevision,
+        'distribRevision': distribRevision,
         'dockerExtraOpts' : [
             "-v /srv/salt/reclass:/srv/salt/reclass:ro",
             "-v /etc/salt:/etc/salt:ro",
@@ -237,13 +265,11 @@
 
 def testNode(LinkedHashMap config) {
     def common = new com.mirantis.mk.Common()
-    def result = ''
     def dockerHostname = config.get('dockerHostname')
     def reclassEnv = config.get('reclassEnv')
     def clusterName = config.get('clusterName', "")
     def formulasSource = config.get('formulasSource', 'pkg')
     def extraFormulas = config.get('extraFormulas', 'linux')
-    def reclassVersion = config.get('reclassVersion', 'master')
     def ignoreClassNotfound = config.get('ignoreClassNotfound', false)
     def aptRepoUrl = config.get('aptRepoUrl', "")
     def aptRepoGPG = config.get('aptRepoGPG', "")
@@ -252,10 +278,9 @@
         "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5",
         "MASTER_HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
         "MINION_ID=${dockerHostname}", "FORMULAS_SOURCE=${formulasSource}",
-        "EXTRA_FORMULAS=${extraFormulas}", "RECLASS_VERSION=${reclassVersion}",
+        "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
         "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1",
-        "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}",
-        "EXTRA_FORMULAS_PKG_ALL=true"
+        "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}"
     ]
 
     config['runCommands'] = [
@@ -265,11 +290,15 @@
 
         '002_Prepare_something'          : {
             sh('''rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2  salt' >> /etc/hosts
-              cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
-              cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
+              cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
+              cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
             ''')
         },
 
+        '003_Install_Reclass_package'    : {
+            sh('apt-get install -y reclass')
+        },
+
         '004_Run_tests'                  : {
             def testTimeout = 40 * 60
             timeout(time: testTimeout, unit: 'SECONDS') {
@@ -385,8 +414,8 @@
             """)
                     sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
                     sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2  salt' >> /etc/hosts
-            cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
-            cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
+            cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
+            cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
             """)
                     // FIXME: should be changed to use reclass from mcp_extra_nigtly?
                     sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do