Updated ping command to work with MCC/MOS
- updated Pinger class with inherited structure for Salt and Kube
- implemented DeamonSet handling in KubeApi interface
- implemented put-textfile and series of ConfigMap methods in KubeApi
- updated Pinger to use multiple --cidr commands at once
- update Summary section to be more informative and human readable
Change-Id: Iac18a619d0bb9a36a286a07f38aeba8f99a454ca
Related-PROD: PROD-36603
diff --git a/cfg_checker/common/decorators.py b/cfg_checker/common/decorators.py
new file mode 100644
index 0000000..eed3fba
--- /dev/null
+++ b/cfg_checker/common/decorators.py
@@ -0,0 +1,60 @@
+# Author: https://gist.github.com/FBosler/be10229aba491a8c912e3a1543bbc74e
+# Updated to fit current framework by Alex Savatieiev
+from functools import wraps
+import time
+
+from cfg_checker.common import logger, logger_cli
+
+
+def retry(exceptions, total_tries=4, initial_wait=0.5, backoff_factor=2):
+ """
+ calling the decorated function applying an exponential backoff.
+ Args:
+ exceptions: Exception(s) that trigger a retry, can be a tuple
+ total_tries: Total tries
+ initial_wait: Time to first retry
+ backoff_factor: Backoff multiplier (e.g. value of 2 will double
+ the delay each retry).
+ logger: logger to be used, if none specified print
+ """
+ def retry_decorator(f):
+ @wraps(f)
+ def func_with_retries(*args, **kwargs):
+ _tries, _delay = total_tries + 1, initial_wait
+ while _tries > 1:
+ try:
+ return f(*args, **kwargs)
+ except exceptions as e:
+ _tries -= 1
+ print_args = args if args else "no args"
+ if _tries == 1:
+ msg = "... {} failed after {} tries".format(
+ f.__name___,
+ total_tries
+ )
+ logger_cli.debug(msg)
+ logger.debug(
+ msg + "args: {}, kwargs: {}".format(
+ print_args,
+ kwargs
+ )
+ )
+ raise
+ msg = "... {}; Exception: {}.\n" \
+ "... retrying in {} seconds!".format(
+ f.__name__,
+ e,
+ _delay
+ )
+ logger_cli.debug(msg)
+ logger.debug(
+ msg + "args: {}, kwargs: {}\n".format(
+ print_args,
+ kwargs
+ )
+ )
+ time.sleep(_delay)
+ _delay *= backoff_factor
+
+ return func_with_retries
+ return retry_decorator
diff --git a/cfg_checker/common/kube_utils.py b/cfg_checker/common/kube_utils.py
index 86e59a5..042db5d 100644
--- a/cfg_checker/common/kube_utils.py
+++ b/cfg_checker/common/kube_utils.py
@@ -8,14 +8,17 @@
from kubernetes import client as kclient, config as kconfig
from kubernetes.stream import stream
+from kubernetes.client.rest import ApiException
from cfg_checker.common import logger, logger_cli
+from cfg_checker.common.decorators import retry
from cfg_checker.common.exception import InvalidReturnException, KubeException
from cfg_checker.common.file_utils import create_temp_file_with_content
from cfg_checker.common.other import utils, shell
from cfg_checker.common.ssh_utils import ssh_shell_p
from cfg_checker.common.const import ENV_LOCAL
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -208,7 +211,10 @@
@property
def CoreV1(self):
if not self._coreV1:
- self._coreV1 = kclient.CoreV1Api(self.kApi)
+ if self.is_local:
+ self._coreV1 = kclient.CoreV1Api(kclient.ApiClient())
+ else:
+ self._coreV1 = kclient.CoreV1Api(kclient.ApiClient(self.kConf))
return self._coreV1
@property
@@ -306,7 +312,7 @@
"... searching for pods with the name '{}'".format(pod_name)
)
_pods = {}
- _pods = self._coreV1.list_namespaced_pod(namespace)
+ _pods = self.CoreV1.list_namespaced_pod(namespace)
_names = self._get_listed_attrs(_pods.items, "metadata.name")
_pname = ""
_pnames = [n for n in _names if n.startswith(pod_name)]
@@ -334,11 +340,12 @@
# If not, output gets converted to str
# Which causes to change " to '
# After that json.loads(...) fail
+ cmd = cmd if isinstance(cmd, list) else cmd.split()
_pod_stream = stream(
self.CoreV1.connect_get_namespaced_pod_exec,
_pname,
namespace,
- command=cmd.split(),
+ command=cmd,
stderr=True,
stdin=False,
stdout=True,
@@ -350,7 +357,11 @@
# run for timeout
_pod_stream.run_forever(timeout=_request_timeout)
# read the output
- return _pod_stream.read_stdout()
+ _output = _pod_stream.read_stdout()
+ # Force recreate of api objects
+ self._coreV1 = None
+ # Send output
+ return _output
def ensure_namespace(self, ns):
"""
@@ -487,3 +498,71 @@
# create result list
return []
+
+ @retry(ApiException)
+ def get_pods_for_daemonset(self, ds):
+ # get all pod names for daemonset
+ logger_cli.debug(
+ "... extracting pod names from daemonset '{}'".format(
+ ds.metadata.name
+ )
+ )
+ _ns = ds.metadata.namespace
+ _name = ds.metadata.name
+ _pods = self.CoreV1.list_namespaced_pod(
+ namespace=_ns,
+ label_selector='name={}'.format(_name)
+ )
+ return _pods
+
+ def put_string_buffer_to_pod_as_textfile(
+ self,
+ pod_name,
+ namespace,
+ buffer,
+ filepath,
+ _request_timeout=120,
+ **kwargs
+ ):
+ _command = ['/bin/sh']
+ response = stream(
+ self.CoreV1.connect_get_namespaced_pod_exec,
+ pod_name,
+ namespace,
+ command=_command,
+ stderr=True,
+ stdin=True,
+ stdout=True,
+ tty=False,
+ _request_timeout=_request_timeout,
+ _preload_content=False,
+ **kwargs
+ )
+
+ # if json
+ # buffer = json.dumps(_dict, indent=2).encode('utf-8')
+
+ commands = [
+ bytes("cat <<'EOF' >" + filepath + "\n", 'utf-8'),
+ buffer,
+ bytes("\n" + "EOF\n", 'utf-8')
+ ]
+
+ while response.is_open():
+ response.update(timeout=1)
+ if response.peek_stdout():
+ logger_cli.debug("... STDOUT: %s" % response.read_stdout())
+ if response.peek_stderr():
+ logger_cli.debug("... STDERR: %s" % response.read_stderr())
+ if commands:
+ c = commands.pop(0)
+ logger_cli.debug("... running command... {}\n".format(c))
+ response.write_stdin(str(c, encoding='utf-8'))
+ else:
+ break
+ response.close()
+
+ # Force recreate of Api objects
+ self._coreV1 = None
+
+ return
diff --git a/cfg_checker/common/salt_utils.py b/cfg_checker/common/salt_utils.py
index f7ea50b..08f2d2b 100644
--- a/cfg_checker/common/salt_utils.py
+++ b/cfg_checker/common/salt_utils.py
@@ -41,8 +41,7 @@
username=config.ssh_user,
keypath=config.ssh_key,
piped=False,
- use_sudo=config.ssh_uses_sudo,
- silent=True
+ use_sudo=config.ssh_uses_sudo
)
if len(_result) < 1:
raise InvalidReturnException(
diff --git a/cfg_checker/common/settings.py b/cfg_checker/common/settings.py
index 993f92f..7b79354 100644
--- a/cfg_checker/common/settings.py
+++ b/cfg_checker/common/settings.py
@@ -327,7 +327,8 @@
self.kube_config_detected = True
else:
logger_cli.debug("... KUBECONFIG env var not found")
- self.kube_config_path = None
+ # do not change it from default
+ # self.kube_config_path = None
self.kube_config_detected = False
else:
logger_cli.debug(
@@ -339,7 +340,7 @@
# try to load values from KUBECONF
_kube_conf = None
- if self.kube_config_path:
+ if self.kube_config_path and self.kube_config_detected:
with open(self.kube_config_path) as kF:
_kube_conf = yaml.load(kF, Loader=yaml.SafeLoader)
# extract host ip