grains / crush / auto generate keyrings
Change-Id: I380d2d0dad799da0b1cc55b16517402b60421a30
diff --git a/README.rst b/README.rst
index 6a62baf..579ac19 100644
--- a/README.rst
+++ b/README.rst
@@ -316,12 +316,14 @@
- dev: /dev/sdc
block_db: /dev/ssd
block_wal: /dev/ssd
+ class: ssd
+ weight: 1.666
- dev: /dev/sdd
enabled: false
-Ceph client roles
------------------
+Ceph client roles - ...Deprecated - use ceph:common instead
+--------------------------------------------------------
Simple ceph client service
diff --git a/_grains/ceph.py b/_grains/ceph.py
new file mode 100644
index 0000000..1b4dfb2
--- /dev/null
+++ b/_grains/ceph.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+
+def main():
+
+ from subprocess import check_output
+ import shlex
+ import os
+ import re
+
+ # osd
+ mount_path = check_output("df -h | awk '{print $6}' | grep ceph | sed 's/[0-9]*//g' | awk 'NR==1{print $1}'", shell=True).rstrip()
+ sed = 'sed \'s#{0}##g\''.format(mount_path)
+ cmd = "df -h | awk '{print $1,$6}' | grep ceph | " + sed
+ osd_output = check_output(cmd, shell=True)
+ grain = {}
+ grain["ceph"] = {}
+ if osd_output:
+ devices = {}
+ for line in osd_output.splitlines():
+ device = line.split()
+ dev = device[0].replace('1','')
+ device[0] = device[1]
+ devices[device[0]] = {}
+ devices[device[0]]['dev'] = dev
+ tline = check_output("ceph osd tree | awk '{print $1,$2,$3,$4}' | grep -w 'osd." + device[0] + "'", shell=True)
+ osd = tline.split()
+ crush_class = osd[1]
+ crush_weight = osd[2]
+ devices[device[0]]['class'] = crush_class
+ devices[device[0]]['weight'] = crush_weight
+ grain["ceph"]["ceph_disk"] = devices
+
+ # keyrings
+ directory = '/etc/ceph/'
+ keyrings = {}
+ if os.path.isdir(directory):
+ for filename in os.listdir(directory):
+ if filename.endswith(".keyring") and filename.startswith("ceph.client"):
+ keyring_output = open(os.path.join(directory, filename), "r")
+ keyring_name = re.search('ceph.client.(.+?).keyring', filename).group(1)
+ if keyring_output:
+ keyrings[keyring_name] = {}
+ for line in keyring_output:
+ attr = shlex.split(line)
+ if attr:
+ if attr[0] == 'key':
+ keyrings[keyring_name]['key'] = attr[2]
+ if attr[0] == 'caps' and 'caps' in keyrings[keyring_name]:
+ keyrings[keyring_name]['caps'][attr[1]] = attr[3]
+ elif attr[0] == 'caps' and 'caps' not in keyrings[keyring_name]:
+ keyrings[keyring_name]['caps'] = {}
+ keyrings[keyring_name]['caps'][attr[1]] = attr[3]
+ if keyrings:
+ grain["ceph"]["ceph_keyring"] = keyrings
+
+ # mon keyring
+ hostname = check_output("hostname", shell=True).rstrip()
+ filepath = "/var/lib/ceph/mon/ceph-{0}/keyring".format(hostname)
+ if os.path.isfile(filepath):
+ mon_key_output = open(filepath, "r")
+ if mon_key_output:
+ keyrings['mon'] = {}
+ for line in mon_key_output:
+ attr = shlex.split(line)
+ if attr:
+ if attr[0] == 'key':
+ keyrings['mon']['key'] = attr[2]
+ if attr[0] == 'caps' and 'caps' in keyrings['mon']:
+ keyrings['mon']['caps'][attr[1]] = attr[3]
+ elif attr[0] == 'caps' and 'caps' not in keyrings['mon']:
+ keyrings['mon']['caps'] = {}
+ keyrings['mon']['caps'][attr[1]] = attr[3]
+ grain["ceph"]["ceph_keyring"] = keyrings
+
+ return grain
diff --git a/ceph/common.sls b/ceph/common.sls
index d3ff157..0e4fb1a 100644
--- a/ceph/common.sls
+++ b/ceph/common.sls
@@ -1,25 +1,38 @@
{%- from "ceph/map.jinja" import common with context %}
-base_packages:
+common_packages:
pkg.installed:
- names: {{ common.pkgs }}
-base_config:
+common_config:
file.managed:
- name: /etc/ceph/ceph.conf
- source: salt://ceph/files/{{ common.version }}/ceph.conf.{{ grains.os_family }}
- template: jinja
- require:
- - pkg: base_packages
+ - pkg: common_packages
{% for keyring_name, keyring in common.get('keyring', {}).iteritems() %}
-/etc/ceph/ceph.client.{{ keyring_name }}.keyring:
+{%- if keyring_name == 'admin' and keyring.key is undefined %}
+
+ceph_create_keyring_{{ keyring_name }}:
+ cmd.run:
+ - name: "ceph-authtool --create-keyring /etc/ceph/ceph.client.{{ keyring_name }}.keyring --gen-key -n client.{{ keyring_name }} {%- for cap_name, cap in keyring.caps.iteritems() %} --cap {{ cap_name }} '{{ cap }}' {%- endfor %}"
+ - unless: "test -f /etc/ceph/ceph.client.{{ keyring_name }}.keyring"
+ - require:
+ - pkg: common_packages
+ - file: common_config
+
+{%- endif %}
+
+{% endfor %}
+
+/etc/ceph/ceph.client.admin.keyring:
file.managed:
- source: salt://ceph/files/keyring
- template: jinja
- - defaults:
- name: {{ keyring_name }}
- keyring: {{ keyring }}
-
-{% endfor %}
+ - unless: "test -f /etc/ceph/ceph.client.admin.keyring"
+ - require:
+ - pkg: common_packages
+ - file: common_config
diff --git a/ceph/files/keyring b/ceph/files/keyring
index e595a2c..6ec9e3e 100644
--- a/ceph/files/keyring
+++ b/ceph/files/keyring
@@ -1,6 +1,21 @@
+{%- for node_name, node_grains in salt['mine.get']('ceph:common:keyring:admin', 'grains.items', 'pillar').iteritems() %}
+
+{%- if node_grains.ceph is defined and node_grains.ceph.ceph_keyring is defined %}
+
+{%- for name, keyring in node_grains.ceph.get("ceph_keyring", {}).iteritems() %}
+
+{%- if name == 'admin' %}
[client.{{ name }}]
key = {{ keyring.key }}
{%- for cap_name, cap in keyring.caps.iteritems() %}
caps {{ cap_name }} = "{{ cap }}"
{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
diff --git a/ceph/files/mon_keyring b/ceph/files/mon_keyring
index e0161dd..ead488e 100644
--- a/ceph/files/mon_keyring
+++ b/ceph/files/mon_keyring
@@ -1,7 +1,21 @@
-{%- from "ceph/map.jinja" import mon with context %}
+{%- for node_name, node_grains in salt['mine.get']('ceph:mon:keyring:mon', 'grains.items', 'pillar').iteritems() %}
+
+{%- if node_grains.ceph is defined and node_grains.ceph.ceph_keyring is defined %}
+
+{%- for name, keyring in node_grains.ceph.get("ceph_keyring", {}).iteritems() %}
+
+{%- if name == 'mon' %}
[mon.]
- key = {{ mon.keyring.mon.key }}
- {%- for cap_name, cap in mon.keyring.mon.caps.iteritems() %}
+ key = {{ keyring.key }}
+ {%- for cap_name, cap in keyring.caps.iteritems() %}
caps {{ cap_name }} = "{{ cap }}"
{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
diff --git a/ceph/mgr.sls b/ceph/mgr.sls
index 5d5e271..8d86ab3 100644
--- a/ceph/mgr.sls
+++ b/ceph/mgr.sls
@@ -36,6 +36,19 @@
- require:
- file: /var/lib/ceph/mgr/ceph-{{ grains.host }}/
+mgr_services:
+ service.running:
+ - enable: true
+ - names: [ceph-mgr@{{ grains.host }}]
+ - watch:
+ - file: /etc/ceph/ceph.conf
+ - require:
+ - pkg: mon_packages
+ - cmd: ceph_create_mgr_keyring_{{ grains.host }}
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
{%- if mgr.get('dashboard', {}).get('enabled', False) %}
ceph_dashboard_address:
@@ -72,18 +85,4 @@
{%- endif %}
-mon_services:
- service.running:
- - enable: true
- - names: [ceph-mgr@{{ grains.host }}]
- - watch:
- - file: /etc/ceph/ceph.conf
- - require:
- - pkg: mon_packages
- - cmd: ceph_create_mgr_keyring_{{ grains.host }}
- {%- if grains.get('noservices') %}
- - onlyif: /bin/false
- {%- endif %}
-
-
{%- endif %}
\ No newline at end of file
diff --git a/ceph/mon.sls b/ceph/mon.sls
index 53dab96..262095d 100644
--- a/ceph/mon.sls
+++ b/ceph/mon.sls
@@ -14,21 +14,26 @@
- require:
- pkg: mon_packages
-cluster_secret_key:
+cluster_{{ grains.host }}_secret_key:
cmd.run:
- name: "ceph-authtool --create-keyring /etc/ceph/ceph.mon.{{ grains.host }}.keyring --gen-key -n mon. --cap mon 'allow *'"
- unless: "test -f /etc/ceph/ceph.mon.{{ grains.host }}.keyring"
+ - require:
+ - pkg: mon_packages
add_admin_keyring_to_mon_keyring:
cmd.run:
- name: "ceph-authtool /etc/ceph/ceph.mon.{{ grains.host }}.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring"
- unless: "test -f /var/lib/ceph/mon/ceph-{{ grains.host }}/done"
+ - require:
+ - pkg: mon_packages
generate_monmap:
cmd.run:
- name: "monmaptool --create {%- for member in common.members %} --add {{ member.name }} {{ member.host }} {%- endfor %} --fsid {{ common.fsid }} /tmp/monmap"
- unless: "test -f /tmp/monmap"
-
+ - require:
+ - pkg: mon_packages
#/var/lib/ceph/mon/ceph-{{ grains.host }}:
# file.directory:
@@ -42,32 +47,64 @@
- user: ceph
- group: ceph
- replace: false
-
+ - require:
+ - pkg: mon_packages
populate_monmap:
cmd.run:
- name: "sudo -u ceph ceph-mon --mkfs -i {{ grains.host }} --monmap /tmp/monmap"
- unless: "test -f /var/lib/ceph/mon/ceph-{{ grains.host }}/kv_backend"
+ - require:
+ - pkg: mon_packages
+
+{% for keyring_name, keyring in mon.get('keyring', {}).iteritems() %}
+
+{%- if keyring_name == 'mon' and keyring.key is undefined %}
+
+cluster_secret_key:
+ cmd.run:
+ - name: "ceph-authtool --create-keyring /var/lib/ceph/mon/ceph-{{ grains.host }}/keyring --gen-key -n mon. {%- for cap_name, cap in keyring.caps.iteritems() %} --cap {{ cap_name }} '{{ cap }}' {%- endfor %}"
+ - unless: "test -f /var/lib/ceph/mon/ceph-{{ grains.host }}/done"
+ - require:
+ - pkg: mon_packages
+
+cluster_secret_key_flag:
+ file.managed:
+ - name: /var/lib/ceph/mon/ceph-{{ grains.host }}/done
+ - user: ceph
+ - group: ceph
+ - content: { }
+ - require:
+ - pkg: mon_packages
+
+{%- endif %}
+
+{% endfor %}
/var/lib/ceph/mon/ceph-{{ grains.host }}/keyring:
file.managed:
- source: salt://ceph/files/mon_keyring
- template: jinja
+ - unless: "test -f /var/lib/ceph/mon/ceph-{{ grains.host }}/done"
+ - require:
+ - pkg: mon_packages
/var/lib/ceph/mon/ceph-{{ grains.host }}/done:
file.managed:
- - user: ceph
- - group: ceph
- - content: { }
+ - user: ceph
+ - group: ceph
+ - content: { }
+ - require:
+ - pkg: mon_packages
mon_services:
service.running:
- - enable: true
- - names: [ceph-mon@{{ grains.host }}]
- - watch:
- - file: /etc/ceph/ceph.conf
- - require:
- - pkg: mon_packages
- {%- if grains.get('noservices') %}
- - onlyif: /bin/false
- {%- endif %}
+ - enable: true
+ - names: [ceph-mon@{{ grains.host }}]
+ - watch:
+ - file: /etc/ceph/ceph.conf
+ - require:
+ - pkg: mon_packages
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
diff --git a/ceph/osd/custom.sls b/ceph/osd/custom.sls
new file mode 100644
index 0000000..1410559
--- /dev/null
+++ b/ceph/osd/custom.sls
@@ -0,0 +1,49 @@
+{%- from "ceph/map.jinja" import osd with context %}
+
+{% set ceph_version = pillar.ceph.common.version %}
+
+{%- for backend_name, backend in osd.backend.iteritems() %}
+
+{%- for disk in backend.disks %}
+
+{%- if disk.get('enabled', True) %}
+
+{% set dev = disk.dev %}
+
+{%- for disk_id, ceph_disk in salt['grains.get']('ceph:ceph_disk', {}).iteritems() %}
+
+{%- if ceph_disk.get('dev') == dev %}
+
+{%- if disk.class is defined %}
+
+update_class_disk_{{ dev }}:
+ cmd.run:
+ - name: "ceph osd crush rm-device-class osd.{{ disk_id }}; ceph osd crush set-device-class {{ disk.class }} osd.{{ disk_id }}"
+ - unless: "ceph osd tree | awk '{print $2,$4}' | grep -w osd.{{ disk_id }} | grep {{ disk.class }}"
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
+{%- endif %}
+
+{%- if disk.weight is defined %}
+
+update_weight_disk_{{ dev }}:
+ cmd.run:
+ - name: "ceph osd crush reweight osd.{{ disk_id }} {{ disk.weight }}"
+ - unless: "ceph osd tree | awk '{print $3,$4}' | grep -w osd.{{ disk_id }} | grep {{ disk.weight }}"
+ {%- if grains.get('noservices') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
+{%- endif %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endfor %}
diff --git a/ceph/osd/init.sls b/ceph/osd/init.sls
new file mode 100644
index 0000000..0813222
--- /dev/null
+++ b/ceph/osd/init.sls
@@ -0,0 +1,4 @@
+include:
+- ceph.common
+- ceph.setup.keyring
+- ceph.osd.setup
diff --git a/ceph/osd.sls b/ceph/osd/setup.sls
similarity index 98%
rename from ceph/osd.sls
rename to ceph/osd/setup.sls
index 9f6c19f..79d7e9f 100644
--- a/ceph/osd.sls
+++ b/ceph/osd/setup.sls
@@ -1,8 +1,5 @@
{%- from "ceph/map.jinja" import osd, common with context %}
-include:
-- ceph.common
-
ceph_osd_packages:
pkg.installed:
- names: {{ osd.pkgs }}
@@ -162,4 +159,3 @@
{%- if grains.get('noservices') %}
- onlyif: /bin/false
{%- endif %}
-
diff --git a/ceph/setup/keyring.sls b/ceph/setup/keyring.sls
index 702200c..4f4865f 100644
--- a/ceph/setup/keyring.sls
+++ b/ceph/setup/keyring.sls
@@ -2,9 +2,13 @@
{% for keyring_name, keyring in common.get('keyring', {}).iteritems() %}
-ceph_keyring_{{ keyring_name }}_import:
- cmd.run:
- - name: ceph auth import -i /etc/ceph/ceph.client.{{ keyring_name }}.keyring
- - unless: ceph auth list | grep {{ keyring_name }}
+{%- if keyring_name != 'admin' %}
-{%- endfor %}
+ceph_create_keyring_{{ keyring_name }}:
+ cmd.run:
+ - name: "ceph auth get-or-create client.{{ keyring_name }} {%- for cap_name, cap in keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %} > /etc/ceph/ceph.client.{{ keyring_name }}.keyring"
+ - unless: "test -f /etc/ceph/ceph.client.{{ keyring_name }}.keyring"
+
+{%- endif %}
+
+{% endfor %}
diff --git a/metadata/service/common/cluster.yml b/metadata/service/common/cluster.yml
index 9e729fe..2358610 100644
--- a/metadata/service/common/cluster.yml
+++ b/metadata/service/common/cluster.yml
@@ -15,11 +15,3 @@
host: ${_param:ceph_mon_node02_address}
- name: ${_param:ceph_mon_node03_hostname}
host: ${_param:ceph_mon_node03_address}
- keyring:
- admin:
- key: ${_param:ceph_admin_keyring}
- caps:
- mds: "allow *"
- mgr: "allow *"
- mon: "allow *"
- osd: "allow *"
diff --git a/metadata/service/mon/cluster.yml b/metadata/service/mon/cluster.yml
index 6639484..6a10da2 100644
--- a/metadata/service/mon/cluster.yml
+++ b/metadata/service/mon/cluster.yml
@@ -7,15 +7,3 @@
ceph:
mon:
enabled: true
- keyring:
- mon:
- key: ${_param:ceph_mon_keyring}
- caps:
- mon: "allow *"
- admin:
- key: ${_param:ceph_admin_keyring}
- caps:
- mds: "allow *"
- mgr: "allow *"
- mon: "allow *"
- osd: "allow *"
\ No newline at end of file
diff --git a/metadata/service/mon/single.yml b/metadata/service/mon/single.yml
index da43193..8be5da5 100644
--- a/metadata/service/mon/single.yml
+++ b/metadata/service/mon/single.yml
@@ -9,13 +9,13 @@
enabled: true
keyring:
mon:
- key: ${_param:ceph_mon_keyring}
caps:
mon: "allow *"
+ common:
+ keyring:
admin:
- key: ${_param:ceph_admin_keyring}
caps:
mds: "allow *"
mgr: "allow *"
mon: "allow *"
- osd: "allow *"
\ No newline at end of file
+ osd: "allow *"