Wait for mgr to be functional before accessing it
PROD-34796
Change-Id: I6e754c19abc3eb738be29eea6eec2d03457a4532
diff --git a/ceph/map.jinja b/ceph/map.jinja
index 1833052..ae0206b 100644
--- a/ceph/map.jinja
+++ b/ceph/map.jinja
@@ -23,6 +23,7 @@
Debian:
pkgs:
- ceph-mgr
+ - jq
services:
- ceph-mgr
{%- endload %}
diff --git a/ceph/mgr.sls b/ceph/mgr.sls
index 44299fe..f1b51b1 100644
--- a/ceph/mgr.sls
+++ b/ceph/mgr.sls
@@ -53,6 +53,18 @@
- onlyif: /bin/false
{%- endif %}
+ceph_mgr_wait_functional:
+ cmd.run:
+ - name: test "$(ceph mgr dump --connect-timeout 30 | jq .available)" = "true"
+ - retry:
+ attempts: 5
+ interval: 10
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+ - require:
+ - service: mgr_services
+
{%- if common.version not in ['kraken', 'jewel'] %}
{%- if mgr.get('dashboard', {}).get('enabled', False) %}
@@ -75,6 +87,7 @@
cmd.wait:
- name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module disable dashboard;ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module enable dashboard --force"
- watch:
+ - cmd: ceph_mgr_wait_functional
- cmd: ceph_dashboard_address
- cmd: ceph_dashboard_port
- require:
@@ -85,6 +98,7 @@
- name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module enable dashboard"
- unless: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module ls | python -c 'import sys, json; print json.load(sys.stdin)[\"enabled_modules\"] | grep dashboard"
- require:
+ - cmd: ceph_mgr_wait_functional
- file: common_config
{%- else %}
@@ -94,6 +108,7 @@
- name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module disable dashboard"
- unless: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module ls | python -c 'import sys, json; print json.load(sys.stdin)[\"disabled_modules\"] | grep dashboard"
- require:
+ - cmd: ceph_mgr_wait_functional
- file: common_config
- file: /var/lib/ceph/mgr/{{ common.get('cluster_name', 'ceph') }}-{{ grains.host }}/
@@ -101,12 +116,12 @@
{%- if pillar.get('prometheus', {}).get('collector',{}).get("enabled", False) %}
-# PROD-34796: formula needs to wait a moment for mgr to start completely after fresh deployment before trying to enable prometheus plugin
enable_prometheus_plugin:
cmd.run:
- - name: "sleep 60; ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module enable prometheus"
+ - name: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module enable prometheus"
- unless: "ceph -c /etc/ceph/{{ common.get('cluster_name', 'ceph') }}.conf mgr module ls | python -c 'import sys, json; print json.load(sys.stdin)[\"enabled_modules\"] | grep prometheus"
- require:
+ - cmd: ceph_mgr_wait_functional
- file: common_config
- file: /var/lib/ceph/mgr/{{ common.get('cluster_name', 'ceph') }}-{{ grains.host }}/
diff --git a/tests/test-requirements.txt b/tests/test-requirements.txt
new file mode 100644
index 0000000..bc7dc7f
--- /dev/null
+++ b/tests/test-requirements.txt
@@ -0,0 +1,4 @@
+jsonschema
+reno
+setuptools<45.0.0
+msgpack<1.0.0