extend crush map and keyring setup
Change-Id: Idcbfbd7833756346694267ad73d08937688058f4
diff --git a/README.rst b/README.rst
index c368704..8b676c3 100644
--- a/README.rst
+++ b/README.rst
@@ -301,13 +301,11 @@
disks:
- dev: /dev/sdm
enabled: false
- rule: hdd
journal: /dev/ssd
fs_type: xfs
class: bestssd
weight: 1.5
- dev: /dev/sdl
- rule: hdd
journal: /dev/ssd
fs_type: xfs
class: bestssd
@@ -429,7 +427,8 @@
pg_num: 256
pgp_num: 256
type: replicated
- crush_ruleset_name: 0
+ crush_rule: sata
+ application: rbd
Erasure ceph storage pool
@@ -442,53 +441,130 @@
pg_num: 256
pgp_num: 256
type: erasure
- crush_ruleset_name: 0
+ crush_rule: ssd
+ application: rbd
-Generate CRUSH map
---------------------
+Generate CRUSH map - Recommended way
+-----------------------------------
-It is required to define the `type` for crush buckets and these types must start with `root` (top) and end with `host`. OSD daemons will be assigned to hosts according to it's hostname. Weight of the buckets will be calculated according to weight of it's childen.
+It is required to define the `type` for crush buckets and these types must start with `root` (top) and end with `host`. OSD daemons will be assigned to hosts according to it's hostname. Weight of the buckets will be calculated according to weight of it's children.
+
+If the pools that are in use have size of 3 it is best to have 3 children of a specific type in the root CRUSH tree to replicate objects across (Specified in rule steps by 'type region').
.. code-block:: yaml
- ceph:
- setup:
- crush:
- enabled: True
- tunables:
- choose_total_tries: 50
- type:
- - root
- - region
- - rack
- - host
- root:
- - name: root1
- - name: root2
- region:
- - name: eu-1
- parent: root1
- - name: eu-2
- parent: root1
- - name: us-1
- parent: root2
- rack:
- - name: rack01
- parent: eu-1
- - name: rack02
- parent: eu-2
- - name: rack03
- parent: us-1
- rule:
- sata:
- ruleset: 0
- type: replicated
- min_size: 1
- max_size: 10
- steps:
- - take crushroot.performanceblock.satahss.1
- - chooseleaf firstn 0 type failure_domain
- - emit
+ ceph:
+ setup:
+ crush:
+ enabled: True
+ tunables:
+ choose_total_tries: 50
+ choose_local_tries: 0
+ choose_local_fallback_tries: 0
+ chooseleaf_descend_once: 1
+ chooseleaf_vary_r: 1
+ chooseleaf_stable: 1
+ straw_calc_version: 1
+ allowed_bucket_algs: 54
+ type:
+ - root
+ - region
+ - rack
+ - host
+ root:
+ - name: root-ssd
+ - name: root-sata
+ region:
+ - name: eu-1
+ parent: root-sata
+ - name: eu-2
+ parent: root-sata
+ - name: eu-3
+ parent: root-ssd
+ - name: us-1
+ parent: root-sata
+ rack:
+ - name: rack01
+ parent: eu-1
+ - name: rack02
+ parent: eu-2
+ - name: rack03
+ parent: us-1
+ rule:
+ sata:
+ ruleset: 0
+ type: replicated
+ min_size: 1
+ max_size: 10
+ steps:
+ - take take root-ssd
+ - chooseleaf firstn 0 type region
+ - emit
+ ssd:
+ ruleset: 1
+ type: replicated
+ min_size: 1
+ max_size: 10
+ steps:
+ - take take root-sata
+ - chooseleaf firstn 0 type region
+ - emit
+
+
+Generate CRUSH map - Alternative way
+------------------------------------
+
+It's necessary to create per OSD pillar.
+
+.. code-block:: yaml
+
+ ceph:
+ osd:
+ crush:
+ - type: root
+ name: root1
+ - type: region
+ name: eu-1
+ - type: rack
+ name: rack01
+ - type: host
+ name: osd001
+
+
+Apply CRUSH map
+---------------
+
+Before you apply CRUSH map please make sure that settings in generated file in /etc/ceph/crushmap are correct.
+
+.. code-block:: yaml
+
+ ceph:
+ setup:
+ crush:
+ enforce: true
+ pool:
+ images:
+ crush_rule: sata
+ application: rbd
+ volumes:
+ crush_rule: sata
+ application: rbd
+ vms:
+ crush_rule: ssd
+ application: rbd
+
+
+Persist CRUSH map
+--------------------
+
+After the CRUSH map is applied to Ceph it's recommended to persist the same settings even after OSD reboots.
+
+.. code-block:: yaml
+
+ ceph:
+ osd:
+ crush_update: false
+
Ceph monitoring
---------------
diff --git a/ceph/files/jewel/ceph.conf.Debian b/ceph/files/jewel/ceph.conf.Debian
index 100f12c..ec373ca 100644
--- a/ceph/files/jewel/ceph.conf.Debian
+++ b/ceph/files/jewel/ceph.conf.Debian
@@ -61,7 +61,7 @@
[mon.{{ member.name }}]
mon host = {{ member.name }}
mon addr = {{ member.host }}:6789
-{%- if not loop.last %}
+{%- if not loop.last %}
{%- endif %}
{%- endfor %}
@@ -71,8 +71,14 @@
{%- if pillar.ceph.osd is defined %}
[osd]
+{%- if osd.crush is defined %}
+crush location = {% for crush in osd.crush %}{{ crush.type }}={{ crush.name }}{% if not loop.last %} {% endif %}{% endfor %}
+{%- endif %}
+
+osd crush update on start = {{ osd.get('crush_update', 'true') }}
+
{%- if pillar.ceph.osd.journal_size is defined %}
-osd journal size = {{ pillar.ceph.osd.journal_size }}
+osd journal size = {{ osd.journal_size }}
{%- endif %}
{%- for key, value in common.get('config', {}).get('osd', {}).iteritems() %}
diff --git a/ceph/files/kraken/ceph.conf.Debian b/ceph/files/kraken/ceph.conf.Debian
index 100f12c..ec373ca 100644
--- a/ceph/files/kraken/ceph.conf.Debian
+++ b/ceph/files/kraken/ceph.conf.Debian
@@ -61,7 +61,7 @@
[mon.{{ member.name }}]
mon host = {{ member.name }}
mon addr = {{ member.host }}:6789
-{%- if not loop.last %}
+{%- if not loop.last %}
{%- endif %}
{%- endfor %}
@@ -71,8 +71,14 @@
{%- if pillar.ceph.osd is defined %}
[osd]
+{%- if osd.crush is defined %}
+crush location = {% for crush in osd.crush %}{{ crush.type }}={{ crush.name }}{% if not loop.last %} {% endif %}{% endfor %}
+{%- endif %}
+
+osd crush update on start = {{ osd.get('crush_update', 'true') }}
+
{%- if pillar.ceph.osd.journal_size is defined %}
-osd journal size = {{ pillar.ceph.osd.journal_size }}
+osd journal size = {{ osd.journal_size }}
{%- endif %}
{%- for key, value in common.get('config', {}).get('osd', {}).iteritems() %}
diff --git a/ceph/files/luminous/ceph.conf.Debian b/ceph/files/luminous/ceph.conf.Debian
index 0888985..c2ae63c 100644
--- a/ceph/files/luminous/ceph.conf.Debian
+++ b/ceph/files/luminous/ceph.conf.Debian
@@ -71,8 +71,14 @@
{%- if pillar.ceph.osd is defined %}
[osd]
+{%- if osd.crush is defined %}
+crush location = {% for crush in osd.crush %}{{ crush.type }}={{ crush.name }}{% if not loop.last %} {% endif %}{% endfor %}
+{%- endif %}
+
+osd crush update on start = {{ osd.get('crush_update', 'true') }}
+
{%- if pillar.ceph.osd.journal_size is defined %}
-osd journal size = {{ pillar.ceph.osd.journal_size }}
+osd journal size = {{ osd.journal_size }}
{%- endif %}
{%- for key, value in common.get('config', {}).get('osd', {}).iteritems() %}
diff --git a/ceph/setup/crush.sls b/ceph/setup/crush.sls
index cdf1bbe..d2e67df 100644
--- a/ceph/setup/crush.sls
+++ b/ceph/setup/crush.sls
@@ -4,3 +4,51 @@
file.managed:
- source: salt://ceph/files/crushmap
- template: jinja
+
+{%- if setup.crush.get('enforce', False) %}
+
+ceph_compile_crush_map:
+ cmd.run:
+ - name: crushtool -c /etc/ceph/crushmap -o /etc/ceph/crushmap.compiled
+ - unless: "test -f /etc/ceph/crushmap.compiled"
+
+ceph_enforce_crush_map:
+ cmd.run:
+ - name: ceph osd setcrushmap -i /tmp/crush.new; touch /etc/ceph/crushmap.enforced
+ - unless: "test -f /etc/ceph/crushmap.enforced"
+
+{# after crush map is setup enable appplication and crush rule for a pool #}
+
+{%- if setup.pool is defined %}
+
+{%- for pool_name, pool in setup.pool.iteritems() %}
+
+{%- for option_name, option_value in pool.iteritems() %}
+
+{%- if option_name in ['application', 'crush_rule'] %}
+
+{%- if option_name == 'application' %}
+
+ceph_pool_{{ pool_name }}_enable_{{ option_name }}:
+ cmd.run:
+ - name: ceph osd pool {{ option_name }} enable {{ pool_name }} {{ option_value }}
+ - unless: "ceph osd pool {{ option_name }} get {{ pool_name }} | grep '{{ option_value }}'"
+
+{%- else %}
+
+ceph_pool_option_{{ pool_name }}_{{ option_name }}:
+ cmd.run:
+ - name: ceph osd pool set {{ pool_name }} {{ option_name }} {{ option_value }}
+ - unless: "ceph osd pool get {{ pool_name }} {{ option_name }} | grep '{{ option_name }}: {{ option_value }}'"
+
+{%- endif %}
+
+{%- endif %}
+
+{%- endfor %}
+
+{%- endfor %}
+
+{%- endif %}
+
+{%- endif %}
diff --git a/ceph/setup/init.sls b/ceph/setup/init.sls
index 07d3bcf..6e099ae 100644
--- a/ceph/setup/init.sls
+++ b/ceph/setup/init.sls
@@ -3,12 +3,12 @@
include:
- ceph.common
-{%- if setup.get('crush') %}
-- ceph.setup.crush
-{%- endif %}
{%- if setup.get('pool') %}
- ceph.setup.pool
{%- endif %}
+{%- if setup.get('crush') %}
+- ceph.setup.crush
+{%- endif %}
{%- if common.get('keyring') %}
- ceph.setup.keyring
{%- endif %}
diff --git a/ceph/setup/keyring.sls b/ceph/setup/keyring.sls
index 4f4865f..419ed64 100644
--- a/ceph/setup/keyring.sls
+++ b/ceph/setup/keyring.sls
@@ -2,6 +2,19 @@
{% for keyring_name, keyring in common.get('keyring', {}).iteritems() %}
+{%- if keyring.name is defined %}
+
+{%- if keyring.name != 'admin' %}
+
+ceph_create_keyring_{{ keyring.name }}:
+ cmd.run:
+ - name: "ceph auth get-or-create client.{{ keyring.name }} {%- for cap_name, cap in keyring.caps.iteritems() %} {{ cap_name }} '{{ cap }}' {%- endfor %} > /etc/ceph/ceph.client.{{ keyring.name }}.keyring"
+ - unless: "test -f /etc/ceph/ceph.client.{{ keyring.name }}.keyring"
+
+{%- endif %}
+
+{%- else %}
+
{%- if keyring_name != 'admin' %}
ceph_create_keyring_{{ keyring_name }}:
@@ -11,4 +24,6 @@
{%- endif %}
+{%- endif %}
+
{% endfor %}
diff --git a/ceph/setup/pool.sls b/ceph/setup/pool.sls
index be2ace1..19fe080 100644
--- a/ceph/setup/pool.sls
+++ b/ceph/setup/pool.sls
@@ -5,7 +5,7 @@
ceph_pool_create_{{ pool_name }}:
cmd.run:
- name: ceph osd pool create {{ pool_name }} {{ pool.pg_num }}{% if pool.pgp_num is defined %} {{ pool.pgp_num }}{% endif %} {{ pool.type }}{% if pool.erasure_code_profile is defined %} {{ pool.erasure_code_profile }}{% endif %}{% if pool.crush_ruleset_name is defined %} {{ pool.crush_ruleset_name }}{% endif %}{% if pool.expected_num_objects is defined %} {{ pool.expected_num_objects }}{% endif %}
- - unless: "ceph osd pool ls | grep ^{{ pool_name }}"
+ - unless: "ceph osd pool ls | grep \"^{{ pool_name }}$\""
{# We need to ensure pg_num is applied first #}
{%- if pool.get('pg_num') %}
@@ -17,7 +17,7 @@
{%- for option_name, option_value in pool.iteritems() %}
-{%- if option_name not in ['type', 'pg_num'] %}
+{%- if option_name not in ['type', 'pg_num', 'application', 'crush_rule'] %}
ceph_pool_option_{{ pool_name }}_{{ option_name }}:
cmd.run:
@@ -29,4 +29,3 @@
{%- endfor %}
{%- endfor %}
-
diff --git a/tests/pillar/ceph_osd_single.sls b/tests/pillar/ceph_osd_single.sls
index e85d447..3c84d53 100644
--- a/tests/pillar/ceph_osd_single.sls
+++ b/tests/pillar/ceph_osd_single.sls
@@ -34,18 +34,15 @@
disks:
- dev: /dev/sdm
enabled: false
- rule: hdd
journal: /dev/sdn
fs_type: xfs
class: bestssd
weight: 1.5
- dev: /dev/sdl
- rule: hdd
fs_type: xfs
class: bestssd
weight: 1.5
- dev: /dev/sdo
- rule: hdd
journal: /dev/sdo
fs_type: xfs
class: bestssd