src/com/mirantis/mk/salt.groovy

package com.mirantis.mk

/**
 *
 * SaltStack functions
 *
 */

/**
 * Login to Salt API and return auth token
 *
 * @param url            Salt API server URL
 * @param params         Salt connection params
 */
def getSaltToken(url, params) {
    def http = new com.mirantis.mk.http()
    data = [
        'username': params.creds.username,
        'password': params.creds.password.toString(),
        'eauth': 'pam'
    ]
    authToken = http.sendHttpGetRequest("${url}/login", data, ['Accept': '*/*'])['return'][0]['token']
    return authToken
}

/**
 * Salt connection and context parameters
 *
 * @param url            Salt API server URL
 * @param credentialsID  ID of credentials store entry
 */
def createSaltConnection(url, credentialsId) {
    def common = new com.mirantis.mk.common()
    params = [
        "url": url,
        "credentialsId": credentialsId,
        "authToken": null,
        "creds": common.getPasswordCredentials(credentialsId)
    ]
    params["authToken"] = getSaltToken(url, params)

    return params
}

/**
 * Run action using Salt API
 *
 * @param master   Salt connection object
 * @param client   Client type
 * @param target   Target specification, eg. for compound matches by Pillar
 *                 data: ['expression': 'I@openssh:server', 'type': 'compound'])
 * @param function Function to execute (eg. "state.sls")
 * @param args     Additional arguments to function
 * @param kwargs   Additional key-value arguments to function
 */
@NonCPS
def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null) {
    def http = new com.mirantis.mk.http()

    data = [
        'tgt': target.expression,
        'fun': function,
        'client': client,
        'expr_form': target.type,
    ]

    if (batch) {
        data['batch'] = batch
    }

    if (args) {
        data['arg'] = args
    }

    if (kwargs) {
        data['kwarg'] = kwargs
    }

    headers = [
      'X-Auth-Token': "${master.authToken}"
    ]

    return http.sendHttpPostRequest("${master.url}/", data, headers)
}

def getSaltPillar(master, target, pillar) {
    def out = runSaltCommand(master, 'local', target, 'pillar.get', [pillar.replace('.', ':')])
    return out
}

def enforceSaltState(master, target, state, output = false) {
    def run_states
    if (state instanceof String) {
        run_states = state
    } else {
        run_states = state.join(',')
    }

    def out = runSaltCommand(master, 'local', target, 'state.sls', null, [run_states])
    try {
        checkSaltResult(out)
    } finally {
        if (output == true) {
            printSaltResult(out)
        }
    }
    return out
}

def runSaltCmd(master, target, cmd) {
    return runSaltCommand(master, 'local', target, 'cmd.run', null, [cmd])
}

def syncSaltAll(master, target) {
    return runSaltCommand(master, 'local', target, 'saltutil.sync_all')
}

def enforceSaltApply(master, target, output = false) {
    def out = runSaltCommand(master, 'local', target, 'state.highstate')
    try {
        checkSaltResult(out)
    } finally {
        if (output == true) {
            printSaltResult(out)
        }
    }
    return out
}

def generateSaltNodeKey(master, target, host, keysize = 4096) {
    args = [host]
    kwargs = ['keysize': keysize]
    return runSaltCommand(master, 'wheel', target, 'key.gen_accept', null, args, kwargs)
}

def generateSaltNodeMetadata(master, target, host, classes, parameters) {
    args = [host, '_generated']
    kwargs = ['classes': classes, 'parameters': parameters]
    return runSaltCommand(master, 'local', target, 'reclass.node_create', null, args, kwargs)
}

def orchestrateSaltSystem(master, target, orchestrate) {
    return runSaltCommand(master, 'runner', target, 'state.orchestrate', null, [orchestrate])
}

/**
 * Check result for errors and throw exception if any found
 *
 * @param result    Parsed response of Salt API
 */
def checkSaltResult(result) {
    for (entry in result['return']) {
        if (!entry) {
            throw new Exception("Salt API returned empty response: ${result}")
        }
        for (node in entry) {
            for (resource in node.value) {
                if (resource instanceof String || resource.value.result.toString().toBoolean() != true) {
                    throw new Exception("Salt state on node ${node.key} failed: ${node.value}")
                }
            }
        }
    }
}

/**
 * Print Salt run results in human-friendly form
 *
 * @param result        Parsed response of Salt API
 * @param onlyChanges   If true (default), print only changed resources
 * @param raw           Simply pretty print what we have, no additional
 *                      parsing
 */
def printSaltResult(result, onlyChanges = true, raw = false) {
    if (raw == true) {
        print new groovy.json.JsonBuilder(result).toPrettyString()
    } else {
        def out = [:]
        for (entry in result['return']) {
            for (node in entry) {
                out[node.key] = [:]
                for (resource in node.value) {
                    if (resource instanceof String) {
                        out[node.key] = node.value
                    } else if (resource.value.result.toString().toBoolean() == false || resource.value.changes || onlyChanges == false) {
                        out[node.key][resource.key] = resource.value
                    }
                }
            }
        }

        for (node in out) {
            if (node.value) {
                println "Node ${node.key} changes:"
                print new groovy.json.JsonBuilder(node.value).toPrettyString()
            } else {
                println "No changes for node ${node.key}"
            }
        }
    }
}


def runSaltProcessStep(master, tgt, fun, arg = [], batch = null) {
    if (batch) {
        result = runSaltCommand(master, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg)
    }
    else {
        result = runSaltCommand(master, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg)
    }
    echo("${result}")
}


def validateFoundationInfra(master) {
    runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ['salt-key'])
    runSaltProcessStep(master, 'I@salt:minion', 'test.version')
    runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ['reclass-salt --top'])
    runSaltProcessStep(master, 'I@reclass:storage', 'reclass.inventory')
    runSaltProcessStep(master, 'I@salt:minion', 'state.show_top')
}


def installFoundationInfra(master) {
    runSaltProcessStep(master, 'I@salt:master', 'state.sls', ['salt.master,reclass'])
    runSaltProcessStep(master, 'I@linux:system', 'saltutil.refresh_pillar')
    runSaltProcessStep(master, 'I@linux:system', 'saltutil.sync_all')
    runSaltProcessStep(master, 'I@linux:system', 'state.sls', ['linux,openssh,salt.minion,ntp'])
}


def installOpenstackMkInfra(master) {
    // Install keepaliveds
    //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
    runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
    // Check the keepalived VIPs
    runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
    // Install glusterfs
    runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
    //runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.setup'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
    runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
    runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
    runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
    runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
    // Install rabbitmq
    runSaltProcessStep(master, 'I@rabbitmq:server', 'state.sls', ['rabbitmq'])
    // Check the rabbitmq status
    runSaltProcessStep(master, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
    // Install galera
    runSaltProcessStep(master, 'I@galera:master', 'state.sls', ['galera'])
    runSaltProcessStep(master, 'I@galera:slave', 'state.sls', ['galera'])
    // Check galera status
    runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
    runSaltProcessStep(master, 'I@galera:slave', 'mysql.status')
    // Install haproxy
    runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
    runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
    runSaltProcessStep(master, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
    // Install memcached
    runSaltProcessStep(master, 'I@memcached:server', 'state.sls', ['memcached'])
}


def installOpenstackMkControl(master) {
    // setup keystone service
    //runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keystone.server'])
    runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
    // populate keystone services/tenants/roles/users
    runSaltProcessStep(master, 'I@keystone:client', 'state.sls', ['keystone.client'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; keystone service-list'])
    // Install glance and ensure glusterfs clusters
    //runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glance.server'])
    runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'])
    runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glusterfs.client'])
    // Update fernet tokens before doing request on keystone server
    runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
    // Check glance service
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'])
    // Install and check nova service
    //runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['nova'])
    runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'])
    // Install and check cinder service
    //runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['cinder'])
    runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'])
    // Install neutron service
    //runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['neutron'])
    runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'])
    // Install heat service
    //runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'], 1)
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['heat'])
    runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'])
    // Install horizon dashboard
    runSaltProcessStep(master, 'I@horizon:server', 'state.sls', ['horizon'])
    runSaltProcessStep(master, 'I@nginx:server', 'state.sls', ['nginx'])
}


def installOpenstackMkNetwork(master) {
    // Install opencontrail database services
    //runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
    runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail.database'])
    runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'])
    // Install opencontrail control services
    //runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
    runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail'])
    runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'])
    // Provision opencontrail control services
    runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl01 --host_ip 172.16.10.101 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
    runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl02 --host_ip 172.16.10.102 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
    runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl03 --host_ip 172.16.10.103 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
    // Test opencontrail
    runSaltProcessStep(master, 'I@opencontrail:control', 'cmd.run', ['contrail-status'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
    runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
}


def installOpenstackMkCompute(master) {
    // Configure compute nodes
    runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
    runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
    // Provision opencontrail virtual routers
    runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_vrouter.py --host_name cmp01 --host_ip 172.16.10.105 --api_server_ip 172.16.10.254 --oper add --admin_user admin --admin_password workshop --admin_tenant_name admin'])
    runSaltProcessStep(master, 'I@nova:compute', 'system.reboot')
}


def installOpenstackMcpInfra(master) {
    // Comment nameserver
    runSaltProcessStep(master, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
    // Install glusterfs
    runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
    // Install keepalived
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
    runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
    // Check the keepalived VIPs
    runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
    // Setup glusterfs
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
    runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
    runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
    runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
    runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
    // Install haproxy
    runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
    runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
    // Install docker
    runSaltProcessStep(master, 'I@docker:host', 'state.sls', ['docker.host'])
    runSaltProcessStep(master, 'I@docker:host', 'cmd.run', ['docker ps'])
    // Install bird
    runSaltProcessStep(master, 'I@bird:server', 'state.sls', ['bird'])
    // Install etcd
    runSaltProcessStep(master, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
    runSaltProcessStep(master, 'I@etcd:server', 'cmd.run', ['etcdctl cluster-health'])
}


def installOpenstackMcpControl(master) {

    // Install Kubernetes pool and Calico
    runSaltProcessStep(master, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
    runSaltProcessStep(master, 'I@kubernetes:pool', 'cmd.run', ['calicoctl node status'])

    // Setup etcd server
    runSaltProcessStep(master, 'I@kubernetes:master', 'state.sls', ['etcd.server.setup'])

    // Run k8s without master.setup
    runSaltProcessStep(master, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])

    // Run k8s master setup
    runSaltProcessStep(master, 'ctl01*', 'state.sls', ['kubernetes.master.setup'])

    // Revert comment nameserver
    runSaltProcessStep(master, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])

    // Set route
    runSaltProcessStep(master, 'I@kubernetes:pool', 'cmd.run', ['ip r a 10.254.0.0/16 dev ens4'])

    // Restart kubelet
    runSaltProcessStep(master, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
}


def installOpenstackMcpCompute(master) {
    // Install opencontrail
    runSaltProcessStep(master, 'I@opencontrail:compute', 'state.sls', ['opencontrail'])
    // Reboot compute nodes
    runSaltProcessStep(master, 'I@opencontrail:compute', 'system.reboot')
}


def installStacklightControl(master) {
    runSaltProcessStep(master, 'I@elasticsearch:server', 'state.sls', ['elasticsearch.server'])
    runSaltProcessStep(master, 'I@influxdb:server', 'state.sls', ['influxdb'])
    runSaltProcessStep(master, 'I@kibana:server', 'state.sls', ['kibana.server'])
    runSaltProcessStep(master, 'I@grafana:server', 'state.sls', ['grafana'])
    runSaltProcessStep(master, 'I@nagios:server', 'state.sls', ['nagios'])
    runSaltProcessStep(master, 'I@elasticsearch:client', 'state.sls', ['elasticsearch.client'])
    runSaltProcessStep(master, 'I@kibana:client', 'state.sls', ['kibana.client'])
}

/**
 * Run e2e conformance tests
 *
 * @param k8s_api    Kubernetes api address
 * @param image      Docker image with tests
 */
def runConformanceTests(master, k8s_api, image) {
    runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["docker run --rm --net=host -e API_SERVER=${k8s_api} ${image} >> e2e-conformance.log"])
}

/**
 * Print Salt state run results in human-friendly form
 *
 * @param result        Parsed response of Salt API
 * @param onlyChanges   If true (default), print only changed resources
 *                      parsing
 */
def printSaltStateResult(result, onlyChanges = true) {
    def out = [:]
    for (entry in result['return']) {
        for (node in entry) {
            out[node.key] = [:]
            for (resource in node.value) {
                if (resource instanceof String) {
                    out[node.key] = node.value
                } else if (resource.value.result.toString().toBoolean() == false || resource.value.changes || onlyChanges == false) {
                    out[node.key][resource.key] = resource.value
                }
            }
        }
    }

    for (node in out) {
        if (node.value) {
            println "Node ${node.key} changes:"
            print new groovy.json.JsonBuilder(node.value).toPrettyString()
        } else {
            println "No changes for node ${node.key}"
        }
    }
}

/**
 * Print Salt state run results in human-friendly form
 *
 * @param result        Parsed response of Salt API
 * @param onlyChanges   If true (default), print only changed resources
 *                      parsing
 */
def printSaltCommandResult(result, onlyChanges = true) {
    def out = [:]
    for (entry in result['return']) {
        for (node in entry) {
            out[node.key] = [:]
            for (resource in node.value) {
                out[node.key] = node.value
            }
        }
    }

    for (node in out) {
        if (node.value) {
            println "Node ${node.key} changes:"
            print new groovy.json.JsonBuilder(node.value).toPrettyString()
        } else {
            println "No changes for node ${node.key}"
        }
    }
}