From f9ede3488d97b79439c0d04a847db14d750b8f6b Mon Sep 17 00:00:00 2001 From: Mateusz Matuszkowiak Date: Mon, 8 Oct 2018 13:49:01 +0200 Subject: [PATCH] Retry on index create in ES Additionally, before index creation and index template upgrade check if the ES cluster status is green. Related-Bug: PROD-21858 Change-Id: Iecedbd574a9240be86a20c18a11b3eb668f982be --- elasticsearch/client/init.sls | 11 +++++++++++ elasticsearch/client/update_index_templates.sls | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/elasticsearch/client/init.sls b/elasticsearch/client/init.sls index cd6f2fa..18f4ffc 100644 --- a/elasticsearch/client/init.sls +++ b/elasticsearch/client/init.sls @@ -13,6 +13,17 @@ include: {%- endif %} {%- for index_name, index in client.get('index', {}).iteritems() %} +# TODO: "Replace with module.run when bug is fixed". +# Since bug in retry logic in salt (#49895) we need to use a workaround for now. +elasticsearch_check_cluster_status_before_creating_index_{{ index_name }}: + cmd.run: + - name: curl -sf {{ client.server.host }}:{{ client.server.port }}/_cat/health | awk '{print $4}' | grep green + - retry: + attempts: 5 + until: True + interval: 10 + splay: 5 + elasticsearch_index_{{ index_name }}: {%- if index.get('enabled', False) %} diff --git a/elasticsearch/client/update_index_templates.sls b/elasticsearch/client/update_index_templates.sls index 88c8e64..605c064 100644 --- a/elasticsearch/client/update_index_templates.sls +++ b/elasticsearch/client/update_index_templates.sls @@ -5,6 +5,17 @@ include: - elasticsearch.client.service {%- for index_name, index in client.get('index', {}).iteritems() %} +# TODO: "Replace with module.run when bug is fixed". +# Since bug in retry logic in salt (#49895) we need to use a workaround for now. +elasticsearch_check_cluster_status_before_index_template_upgrade_{{ index_name }}: + cmd.run: + - name: curl -sf {{ client.server.host }}:{{ client.server.port }}/_cat/health | awk '{print $4}' | grep green + - retry: + attempts: 5 + until: True + interval: 10 + splay: 5 + elasticsearch_index_template_upgrade_{{ index_name }}: {%- if index.get('enabled', False) %} -- 2.32.7