Fixes for index errors
Added fixes for not updating metrics in case of:
- NotFoundError issue - now metrics set to 0
- ElasticsearchException - now metrics are dropped
Change-Id: I3b3ab20b1457d4556c21186364b4be16f92cedd8
Related-bug: PROD-27906 (PROD:27906)
diff --git a/prometheus_es_exporter/__init__.py b/prometheus_es_exporter/__init__.py
index b723bfd..6d8c233 100644
--- a/prometheus_es_exporter/__init__.py
+++ b/prometheus_es_exporter/__init__.py
@@ -10,7 +10,8 @@
from collections import OrderedDict
from elasticsearch import Elasticsearch
-from elasticsearch.exceptions import ConnectionTimeout
+from elasticsearch.exceptions import (ConnectionTimeout,
+ ElasticsearchException, NotFoundError)
from functools import partial
from jog import JogFormatter
from prometheus_client import start_http_server, Gauge
@@ -110,15 +111,37 @@
yield gauge
+def zero_gauges(query_name):
+ for metric_name, values in gauges.items():
+ if metric_name.startswith(query_name):
+ label_values, gauge = values
+ for labels in label_values:
+ if labels:
+ gauge.labels(*labels).set(0)
+ else:
+ gauge.set(0)
+
+
+def drop_gauges(query_name):
+ for metric_name in gauges.keys():
+ if metric_name.startswith(query_name):
+ del gauges[metric_name]
+
+
def run_query(es_client, name, indices, query, timeout):
try:
response = es_client.search(index=indices, body=query, request_timeout=timeout)
metrics = parse_response(response, [name])
- except Exception:
- logging.exception('Error while querying indices [%s], query [%s].', indices, query)
- else:
- update_gauges(metrics)
+ except ElasticsearchException as e:
+ if isinstance(e, NotFoundError):
+ logging.warn('Not found indices [%s]. Zeroing metrics for query [%s].', indices, name)
+ zero_gauges(name)
+ else:
+ logging.exception('Error while querying indices [%s], query [%s]. Dropping related metrics.', indices, query)
+ drop_gauges(name)
+ return
+ update_gauges(metrics)
def collector_up_gauge(name_list, description, succeeded=True):