cfg_checker/common/salt_utils.py

#    Author: Alex Savatieiev (osavatieiev@mirantis.com; a.savex@gmail.com)
#    Copyright 2019-2022 Mirantis, Inc.
"""
Module to handle interaction with salt
"""
import json
import os
import time

import requests

from cfg_checker.common import logger, logger_cli
from cfg_checker.common.exception import InvalidReturnException, SaltException
from cfg_checker.common.other import shell
from cfg_checker.common.ssh_utils import ssh_shell_p


def _extract_salt_password(_raw):
    if not isinstance(_raw, str):
        raise InvalidReturnException(_raw)
    else:
        try:
            _json = json.loads(_raw)
        except ValueError:
            raise SaltException(
                "# Return value is not a json: '{}'".format(_raw)
            )

    return _json["local"]


def get_remote_salt_env_password(config):
    """Uses ssh call with configured options to get password from salt master

    :return: password string
    """
    _salt_cmd = "salt-call --out=json pillar.get _param:salt_api_password"
    logger_cli.debug("... calling salt using ssh: '{}'".format(_salt_cmd))
    try:
        _result = ssh_shell_p(
            _salt_cmd,
            config.ssh_host,
            username=config.ssh_user,
            keypath=config.ssh_key,
            piped=False,
            use_sudo=config.ssh_uses_sudo
        )
        if len(_result) < 1:
            raise InvalidReturnException(
                "# Empty value returned for '{}".format(
                    _salt_cmd
                )
            )
        else:
            return _extract_salt_password(_result)
    except OSError as e:
        raise SaltException(
            "Salt error calling '{}': '{}'\n"
            "\nConsider checking 'MCP_ENV' "
            "and '<pkg>/etc/<env>.env' files".format(_salt_cmd, e.strerror)
        )


def get_salt_local_password(config):
    """Calls salt locally to get password from the pillar

    :return: password string
    """
    _cmd = []
    if config.ssh_uses_sudo:
        _cmd = ["sudo"]
    # salt commands
    _cmd.append("salt-call")
    _cmd.append("--out=json pillar.get _param:salt_api_password")
    try:
        _result = shell(" ".join(_cmd))
    except OSError as e:
        raise SaltException(
            "Salt error calling '{}': '{}'\n"
            "\nConsider checking 'MCP_ENV' "
            "and '<pkg>/etc/<env>.env' files".format(_cmd, e.strerror)
        )
    return _extract_salt_password(_result)


def list_to_target_string(node_list, separator):
    result = ''
    for node in node_list:
        result += node + ' ' + separator + ' '
    return result[:-(len(separator)+2)]


class SaltRest(object):
    _auth = {}

    default_headers = {
        'Accept': 'application/json',
        'Content-Type': 'application/json',
        'X-Auth-Token': None
    }

    def __init__(self, config):
        self.config = config

        self._host = config.mcp_host
        self._port = config.salt_port
        self.uri = "http://" + config.mcp_host + ":" + config.salt_port

        self._token = self._login()
        self.last_response = None

    def get(
        self,
        path='',
        headers=default_headers,
        cookies=None,
        timeout=None
    ):
        _path = os.path.join(self.uri, path)
        logger.debug("# GET '{}'\nHeaders: '{}'\nCookies: {}".format(
            _path,
            headers,
            cookies
        ))
        return requests.get(
            _path,
            headers=headers,
            cookies=cookies,
            timeout=timeout
        )

    def post(self, data, path='', headers=default_headers, cookies=None):
        if data is None:
            data = {}
        _path = os.path.join(self.uri, path)
        if path == 'login':
            _data = str(data).replace(self._pass, "*****")
        else:
            _data = data
        logger.debug(
            "# POST '{}'\nHeaders: '{}'\nCookies: {}\nBody: {}".format(
                _path,
                headers,
                cookies,
                _data
            )
        )
        return requests.post(
            os.path.join(self.uri, path),
            headers=headers,
            json=data,
            cookies=cookies
        )

    def _login(self):
        # if there is no password - try to get local, if this available
        if self.config.env_name == "local":
            _pass = get_salt_local_password(self.config)
        else:
            _pass = get_remote_salt_env_password(self.config)
        login_payload = {
            'username': self.config.salt_user,
            'password': _pass,
            'eauth': 'pam'
        }
        self._pass = _pass
        logger.debug("# Logging in to salt master...")
        _response = self.post(login_payload, path='login')

        if _response.ok:
            self._auth['response'] = _response.json()['return'][0]
            self._auth['cookies'] = _response.cookies
            self.default_headers['X-Auth-Token'] = \
                self._auth['response']['token']
            return self._auth['response']['token']
        else:
            raise EnvironmentError(
                "# HTTP:{}, Not authorized?".format(_response.status_code)
            )

    def salt_request(self, fn, *args, **kwargs):
        # if token will expire in 5 min, re-login
        if self._auth['response']['expire'] < time.time() + 300:
            self._auth['response']['X-Auth-Token'] = self._login()

        _method = getattr(self, fn)
        _response = _method(*args, **kwargs)
        self.last_response = _response
        _content = "..."
        _len = len(_response.content)
        if _len < 1024:
            _content = _response.content
        logger.debug(
            "# Response (HTTP {}/{}), {}: {}".format(
                _response.status_code,
                _response.reason,
                _len,
                _content
            )
        )
        if _response.ok:
            return _response.json()['return']
        else:
            raise EnvironmentError(
                "# Salt Error: HTTP:{}, '{}'".format(
                    _response.status_code,
                    _response.reason
                )
            )


class SaltRemote(SaltRest):
    master_node = ""

    def __init__(self, config):
        super(SaltRemote, self).__init__(config)

    def cmd(
            self,
            tgt,
            fun,
            param=None,
            client='local',
            kwarg=None,
            expr_form=None,
            tgt_type=None,
            timeout=None
    ):
        _timeout = timeout if timeout is not None else self.config.salt_timeout
        _payload = {
            'fun': fun,
            'tgt': tgt,
            'client': client,
            'timeout': _timeout
        }

        if expr_form:
            _payload['expr_form'] = expr_form
        if tgt_type:
            _payload['tgt_type'] = tgt_type
        if param:
            _payload['arg'] = param
        if kwarg:
            _payload['kwarg'] = kwarg
        logger_cli.debug("SaltRequest: POST '{}'".format(_payload))
        _response = self.salt_request('post', [_payload])
        if isinstance(_response, list):
            return _response[0]
        else:
            raise EnvironmentError(
                "# Unexpected response from from salt-api/LocalClient: "
                "{}".format(_response)
            )

    def run(self, fun, kwarg=None):
        _payload = {
            'client': 'runner',
            'fun': fun,
            'timeout': self.config.salt_timeout
        }

        if kwarg:
            _payload['kwarg'] = kwarg

        _response = self.salt_request('post', [_payload])
        if isinstance(_response, list):
            return _response[0]
        else:
            raise EnvironmentError(
                "# Unexpected response from from salt-api/RunnerClient: "
                "{}".format(_response)
            )

    def wheel(self, fun, arg=None, kwarg=None):
        _payload = {
            'client': 'wheel',
            'fun': fun,
            'timeout': self.config.salt_timeout
        }

        if arg:
            _payload['arg'] = arg
        if kwarg:
            _payload['kwarg'] = kwarg

        _response = self.salt_request('post', _payload)['data']
        if _response['success']:
            return _response
        else:
            raise EnvironmentError(
                "# Salt Error: '{}'".format(_response['return']))

    def pillar_request(self, node_target, pillar_submodule, argument):
        # example cli: 'salt "ctl01*" pillar.keys rsyslog'
        _type = "compound"
        if isinstance(node_target, list):
            _type = "list"
        return self.cmd(
            node_target,
            "pillar." + pillar_submodule,
            argument,
            expr_form=_type
        )

    def pillar_keys(self, node_target, argument):
        return self.pillar_request(node_target, 'keys', argument)

    def pillar_get(self, node_target, argument):
        return self.pillar_request(node_target, 'get', argument)

    def pillar_data(self, node_target, argument):
        return self.pillar_request(node_target, 'data', argument)

    def pillar_raw(self, node_target, argument):
        return self.pillar_request(node_target, 'raw', argument)

    def list_minions(self):
        """
            Fails in salt version 2016.3.8
            Works starting from 2017.7.7
            api returns dict of minions with grains
        """
        try:
            _r = self.salt_request('get', 'minions', timeout=10)
        except requests.exceptions.ReadTimeout:
            logger_cli.debug("... timeout waiting list minions from Salt API")
            _r = None
        return _r[0] if _r else None

    def list_keys(self):
        """
            Fails in salt version 2016.3.8
            Works starting from 2017.7.7
            api should return dict:
            {
                'local': [],
                'minions': [],
                'minions_denied': [],
                'minions_pre': [],
                'minions_rejected': [],
            }
        """
        return self.salt_request('get', path='keys')

    def get_status(self):
        """
            Fails in salt version 2017.7.7
            'runner' client is the equivalent of 'salt-run'
            Returns the
        """
        return self.run(
            'manage.status',
            kwarg={'timeout': 10}
        )

    def get_active_nodes(self):
        """Used when other minion list metods fail

        :return: json result from salt test.ping
        """
        if self.config.skip_nodes:
            logger.info(
                "# Nodes to be skipped: {0}".format(self.config.skip_nodes)
            )
            _r = self.cmd(
                '* and not ' + list_to_target_string(
                    self.config.skip_nodes,
                    'and not'
                ),
                'test.ping',
                expr_form='compound')
        else:
            _r = self.cmd('*', 'test.ping')
        # Return all nodes that responded
        return [node for node in _r.keys() if _r[node]]

    def get_monitoring_ip(self, param_name):
        salt_output = self.cmd(
            'docker:client:stack:monitoring',
            'pillar.get',
            param=param_name,
            expr_form='pillar')
        return salt_output[salt_output.keys()[0]]

    def f_touch_master(self, path, makedirs=True):
        _kwarg = {
            "makedirs": makedirs
        }
        salt_output = self.cmd(
            self.master_node,
            "file.touch",
            param=path,
            kwarg=_kwarg
        )
        return [*salt_output.values()][0]

    def f_append_master(self, path, strings_list, makedirs=True):
        _kwarg = {
            "makedirs": makedirs
        }
        _args = [path]
        _args.extend(strings_list)
        salt_output = self.cmd(
            self.master_node,
            "file.write",
            param=_args,
            kwarg=_kwarg
        )
        return [*salt_output.values()][0]

    def mkdir(self, target, path, tgt_type=None):
        salt_output = self.cmd(
            target,
            "file.mkdir",
            param=path,
            expr_form=tgt_type
        )
        return salt_output

    def f_manage_file(self, target_path, source,
                      sfn='', ret='{}',
                      source_hash={},
                      user='root', group='root', backup_mode='755',
                      show_diff='base',
                      contents='', makedirs=True):
        """
        REST variation of file.get_managed
        CLI execution goes like this (10 agrs):
        salt cfg01\\* file.manage_file /root/test_scripts/pkg_versions.py
        '' '{}' /root/diff_pkg_version.py
        '{hash_type: 'md5', 'hsum': <md5sum>}' root root '755' base ''
        makedirs=True
            param: name - target file placement when managed
            param: source - source for the file
        """
        _source_hash = {
            "hash_type": "md5",
            "hsum": 000
        }
        _arg = [
            target_path,
            sfn,
            ret,
            source,
            _source_hash,
            user,
            group,
            backup_mode,
            show_diff,
            contents
        ]
        _kwarg = {
            "makedirs": makedirs
        }
        salt_output = self.cmd(
            self.master_node,
            "file.manage_file",
            param=_arg,
            kwarg=_kwarg
        )
        return [*salt_output.values()][0]

    def cache_file(self, target, source_path):
        salt_output = self.cmd(
            target,
            "cp.cache_file",
            param=source_path
        )
        return [*salt_output.values()][0]

    def get_file(self, target, source_path, target_path, tgt_type=None):
        return self.cmd(
            target,
            "cp.get_file",
            param=[source_path, target_path],
            expr_form=tgt_type
        )

    @staticmethod
    def compound_string_from_list(nodes_list):
        return " or ".join(nodes_list)