Fix reshuffle ids during crush map generation
- add support for fixed crush_id for buckets
- start reusing existing id for hosts
Related-Prod: PROD-36797
Change-Id: Ic84626027f5c329ec833280cf5b409284171ae89
diff --git a/_grains/ceph.py b/_grains/ceph.py
index e9f38a6..f080377 100644
--- a/_grains/ceph.py
+++ b/_grains/ceph.py
@@ -26,6 +26,12 @@
attr = shlex.split(line)
grain['ceph']['fsid'] = attr[2]
+ # crush id
+ cmd = "ceph osd crush tree | grep `hostname -s` | awk '{print $1}'"
+ crush_id = check_output(cmd, shell=True).strip()
+ if crush_id:
+ grain['ceph']['crush_id'] = crush_id
+
# osd
if os.path.exists('/var/lib/ceph/osd'):
devices = {}
diff --git a/ceph/files/crushmap b/ceph/files/crushmap
index 2ff78e8..cf12910 100644
--- a/ceph/files/crushmap
+++ b/ceph/files/crushmap
@@ -5,9 +5,10 @@
{%- set hosts = {} -%}
{%- set osds = {} -%}
{%- set weights = {} -%}
+{%- set ids = [0] -%}
-{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() -%}
- {%- if node_grains.get('ceph', {}).get('ceph_disk') and node_grains.get('ceph', {}).get('fsid', '') == common.fsid -%}
+{%- for node_name, node_grains in salt['mine.get']('ceph:osd', 'grains.items', 'pillar').iteritems() -%}
+ {%- if node_grains.get('ceph', {}).get('fsid', '') == common.fsid -%}
{# load OSDs and compute weight#}
{%- set node_weight = [] -%}
{%- for osd_id, osd in node_grains.ceph.ceph_disk.iteritems() -%}
@@ -17,6 +18,11 @@
{%- endfor -%}
{%- do hosts.update({node_grains.nodename: {'weight': node_weight|sum, 'parent': node_grains.ceph_crush_parent }}) -%}
+ {%- if node_grains.ceph.crush_id|d(False) -%}
+ {%- do hosts[node_grains.nodename].update({'crush_id': node_grains.ceph.crush_id|int}) -%}
+ {%- do ids.append(node_grains.ceph.crush_id|int) -%}
+ {%- endif -%}
+
{%- endif -%}
{%- endfor -%}
@@ -24,6 +30,14 @@
{%- set _crush = setup.crush -%}
{%- set _buckets = [] %}
+{%- for i in _crush.type %}
+ {%- for resource in _crush.get(i, {}) -%}
+ {%- if resource.crush_id|d(False) -%}
+ {%- do ids.append(resource.crush_id|int) -%}
+ {%- endif -%}
+ {%- endfor -%}
+{%- endfor %}
+
# tunables
{%- for k, v in _crush.get('tunables', {}).iteritems() %}
tunable {{ k }} {{ v }}
@@ -37,50 +51,50 @@
# devices
{%- set ceph_version = pillar.ceph.common.version -%}
-{%- if ceph_version not in ['kraken', 'jewel'] %}
-{% for disk_id, disk in osds|dictsort %}
-device {{ disk_id }} osd.{{ disk_id }}{%- if disk.get('class') %} class {{ disk.class }}{% endif %}
+{%- if ceph_version not in ['kraken', 'jewel'] -%}
+{% for disk_id in osds|map('int')|sort %}
+device {{ disk_id }} osd.{{ disk_id }}{%- if osds[disk_id|string].get('class') %} class {{ osds[disk_id|string].class }}{% endif %}
{%- endfor %}
{%- else %}
-{% for disk_id, disk in osds|dictsort %}
+{% for disk_id in osds|map('int')|sort %}
device {{ disk_id }} osd.{{ disk_id }}
{%- endfor %}
{%- endif %}
-{% set count = [1] %}
-
# host buckets
{%- for bucket in hosts %}
{%- set weight = [] %}
+{%- set crush_id = hosts[bucket].crush_id | d(ids|min -1) %}
+{%- do ids.append(crush_id|int) %}
host {{ bucket }} {
- id -{{ count[0] }}
+ id {{ crush_id }}
alg straw2
hash 0
{%- for disk_id, disk in osds.iteritems() %}
- {%- if disk.host == bucket %}
+ {%- if disk.host == bucket %}
item osd.{{ disk_id }} weight {{ disk.weight }}
- {%- do weight.append(disk.weight|float) %}
- {%- endif %}
+ {%- do weight.append(disk.weight|float) %}
+ {%- endif %}
{%- endfor %}
{%- do weights.update({'host-' ~ bucket: weight|sum}) %}
# weight: {{ weights['host-' ~ bucket] }}
# parent: {{ hosts[bucket].parent }}
- {% if count.append(count.pop() + 1) %}{% endif %}
}
{%- endfor %}
{%- for i in _crush.type|reverse %}
{%- set _typeid = loop.revindex -%}
{%- for resource in _crush.get(i, {}) -%}
-{% set weight = [] %}
+{%- set weight = [] %}
+{%- set crush_id = resource.crush_id | d(ids|min -1) %}
+{%- do ids.append(crush_id|int) %}
{{ i }} {{ resource.name }} {
- id -{{ count[0] }}
+ id {{ crush_id }}
alg {{ resource.get('alg', 'straw2') }}
hash {{ resource.get('hash', 0) }}
# list children resources - {{ _crush.type[_typeid] }}
{%- if _crush.type[_typeid] == 'host' %}
-
{%- for r_name, r in hosts.iteritems() %}
{%- if r.parent == resource.name %}
item {{ r_name }} weight {{ r.weight }}
@@ -103,7 +117,6 @@
{%- do weights.update({i ~ '-' ~ resource.name: weight|sum }) %}
# weight: {{ weight|sum }}
}
-{% if count.append(count.pop() + 1) %}{% endif %}
{%- endfor %}
{%- endfor %}