From: Tomek Jaroszyk <tjaroszyk@mirantis.com>
Date: Wed, 9 Mar 2022 11:32:50 +0000 (+0100)
Subject: Fix reshuffle ids during crush map generation
X-Git-Url: https://gerrit.mcp.mirantis.com/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F87%2F125887%2F3;p=salt-formulas%2Fceph.git

Fix reshuffle ids during crush map generation
 - add support for fixed crush_id for buckets
 - start reusing existing id for hosts

Related-Prod: PROD-36797
Change-Id: Ic84626027f5c329ec833280cf5b409284171ae89
---

diff --git a/_grains/ceph.py b/_grains/ceph.py
index e9f38a6..f080377 100644
--- a/_grains/ceph.py
+++ b/_grains/ceph.py
@@ -26,6 +26,12 @@ def main():
                     attr = shlex.split(line)
                     grain['ceph']['fsid'] = attr[2]
 
+        # crush id
+        cmd = "ceph osd crush tree | grep `hostname -s` | awk '{print $1}'"
+        crush_id = check_output(cmd, shell=True).strip()
+        if crush_id:
+            grain['ceph']['crush_id'] = crush_id
+
         # osd
         if os.path.exists('/var/lib/ceph/osd'):
             devices = {}
diff --git a/ceph/files/crushmap b/ceph/files/crushmap
index 2ff78e8..cf12910 100644
--- a/ceph/files/crushmap
+++ b/ceph/files/crushmap
@@ -5,9 +5,10 @@
 {%- set hosts = {} -%}
 {%- set osds = {} -%}
 {%- set weights = {} -%}
+{%- set ids = [0] -%}
 
-{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() -%}
-  {%- if node_grains.get('ceph', {}).get('ceph_disk') and node_grains.get('ceph', {}).get('fsid', '') == common.fsid -%}
+{%- for node_name, node_grains in salt['mine.get']('ceph:osd', 'grains.items', 'pillar').iteritems() -%}
+  {%- if node_grains.get('ceph', {}).get('fsid', '') == common.fsid -%}
     {# load OSDs and compute weight#}
     {%- set node_weight = [] -%}
     {%- for osd_id, osd in node_grains.ceph.ceph_disk.iteritems() -%}
@@ -17,6 +18,11 @@
     {%- endfor -%}
 
     {%- do hosts.update({node_grains.nodename: {'weight': node_weight|sum, 'parent': node_grains.ceph_crush_parent }}) -%}
+    {%- if node_grains.ceph.crush_id|d(False) -%}
+      {%- do hosts[node_grains.nodename].update({'crush_id': node_grains.ceph.crush_id|int}) -%}
+      {%- do ids.append(node_grains.ceph.crush_id|int) -%}
+    {%- endif -%}
+
 
   {%- endif -%}
 {%- endfor -%}
@@ -24,6 +30,14 @@
 {%- set _crush = setup.crush -%}
 {%- set _buckets = [] %}
 
+{%- for i in _crush.type %}
+  {%- for resource in _crush.get(i, {}) -%}
+    {%- if resource.crush_id|d(False) -%}
+      {%- do ids.append(resource.crush_id|int) -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- endfor %}
+
 # tunables
 {%- for k, v in _crush.get('tunables', {}).iteritems() %}
 tunable {{ k }} {{ v }}
@@ -37,50 +51,50 @@ type {{ loop.index0 }} {{ i }}
 
 # devices
 {%- set ceph_version = pillar.ceph.common.version -%}
-{%- if ceph_version not in ['kraken', 'jewel'] %}
-{% for disk_id, disk in osds|dictsort %}
-device {{ disk_id }} osd.{{ disk_id }}{%- if disk.get('class') %} class {{ disk.class }}{% endif %}
+{%- if ceph_version not in ['kraken', 'jewel'] -%}
+{% for disk_id in osds|map('int')|sort %}
+device {{ disk_id }} osd.{{ disk_id }}{%- if osds[disk_id|string].get('class') %} class {{ osds[disk_id|string].class }}{% endif %}
 {%- endfor %}
 {%- else %}
-{% for disk_id, disk in osds|dictsort %}
+{% for disk_id in osds|map('int')|sort %}
 device {{ disk_id }} osd.{{ disk_id }}
 {%- endfor %}
 {%- endif %}
 
-{% set count = [1] %}
-
 # host buckets
 {%- for bucket in hosts %}
 {%- set weight = [] %}
+{%- set crush_id = hosts[bucket].crush_id | d(ids|min -1) %}
+{%- do ids.append(crush_id|int) %}
 host {{ bucket }} {
-  id -{{ count[0] }}
+  id {{ crush_id }}
   alg straw2
   hash 0
   {%- for disk_id, disk in osds.iteritems() %}
-  {%- if disk.host == bucket %}
+    {%- if disk.host == bucket %}
   item osd.{{ disk_id }} weight {{ disk.weight }}
-  {%- do weight.append(disk.weight|float) %}
-  {%- endif %}
+      {%- do weight.append(disk.weight|float) %}
+    {%- endif %}
   {%- endfor %}
   {%- do weights.update({'host-' ~ bucket: weight|sum}) %}
   # weight: {{ weights['host-' ~ bucket] }}
   # parent: {{ hosts[bucket].parent }}
-  {% if count.append(count.pop() + 1) %}{% endif %}
 }
 {%- endfor %}
 
 {%- for i in _crush.type|reverse %}
 {%- set _typeid = loop.revindex -%}
 {%- for resource in _crush.get(i, {}) -%}
-{% set weight = [] %}
+{%- set weight = [] %}
+{%- set crush_id = resource.crush_id | d(ids|min -1) %}
+{%- do ids.append(crush_id|int) %}
 {{ i }} {{ resource.name }} {
-  id -{{ count[0] }}
+  id {{ crush_id }}
   alg {{ resource.get('alg', 'straw2') }}
   hash {{ resource.get('hash', 0) }}
 
   # list children resources - {{ _crush.type[_typeid] }}
   {%- if _crush.type[_typeid] == 'host' %}
-
     {%- for r_name, r in hosts.iteritems() %}
       {%- if r.parent == resource.name %}
   item {{ r_name }} weight {{ r.weight }}
@@ -103,7 +117,6 @@ host {{ bucket }} {
   {%- do weights.update({i ~ '-' ~ resource.name: weight|sum }) %}
   # weight: {{ weight|sum }}
 }
-{% if count.append(count.pop() + 1) %}{% endif %}
 {%- endfor %}
 {%- endfor %}