Merge "Add Salt 2018.3 tests"
diff --git a/README.rst b/README.rst
index b7cd315..9cdf64a 100644
--- a/README.rst
+++ b/README.rst
@@ -431,6 +431,10 @@
 .. code-block:: yaml
 
     ceph:
+      common:
+        config:
+          rgw:
+            key: value
       radosgw:
         enabled: true
         hostname: gw.ceph.lab
@@ -718,7 +722,7 @@
         full_backups_to_keep: 3
         incr_before_full: 3
         backup_times:
-          dayOfWeek: 0
+          day_of_week: 0
           hour: 4
           minute: 52
         compression: true
@@ -735,7 +739,7 @@
   ``backup_times`` parameters is not defined, the defalut ``*`` value will be
   used. For example, if minute parameter is ``*``, it will run the backup every minute,
   which is ususally not desired.
-  Available parameters are ``dayOfWeek``, ``dayOfMonth``, ``month``, ``hour`` and ``minute``.
+  Available parameters are ``day_of_week``, ``day_of_month``, ``month``, ``hour`` and ``minute``.
   Please see the crontab reference for further info on how to set these parameters.
 
   .. note:: Please be aware that only ``backup_times`` section OR
@@ -781,7 +785,7 @@
         incr_before_full: 3
         backup_dir: /srv/backup
         backup_times:
-          dayOfWeek: 0
+          day_of_week: 0
           hour: 4
           minute: 52
         key:
@@ -795,7 +799,7 @@
   ``backup_times`` parameters is not defined, the defalut ``*`` value will be
   used. For example, if minute parameter is ``*``, it will run the backup every minute,
   which is ususally not desired.
-  Available parameters are ``dayOfWeek``, ``dayOfMonth``, ``month``, ``hour`` and ``minute``.
+  Available parameters are ``day_of_week``, ``day_of_month``, ``month``, ``hour`` and ``minute``.
   Please see the crontab reference for further info on how to set these parameters.
 
   .. note:: Please be aware that only ``backup_times`` section OR
diff --git a/_grains/ceph.py b/_grains/ceph.py
index 61254bf..ec568c9 100644
--- a/_grains/ceph.py
+++ b/_grains/ceph.py
@@ -4,6 +4,7 @@
 def main():
 
     from subprocess import check_output
+    from subprocess import CalledProcessError
     import shlex
     import os
     import re
@@ -40,20 +41,23 @@
                         dev = device[0].replace('1','')
                         dev = re.sub("\d+", "", device[0])
                     device[0] = device[2]
-                    devices[device[0]] = {}
-                    devices[device[0]]['dev'] = dev
-                    if encrypted:
-                        devices[device[0]]['dmcrypt'] = 'true'
-                    tline = check_output("ceph -c " + conf_file + " osd tree | awk '{print $1,$2,$3,$4}' | grep -w 'osd." + device[0] + "'", shell=True)
-                    osd = tline.split()
-                    if "osd" not in osd[2]:
-                        crush_class = osd[1]
-                        crush_weight = osd[2]
-                        devices[device[0]]['class'] = crush_class
-                        devices[device[0]]['weight'] = crush_weight
-                    else:
-                        crush_weight = osd[1]
-                        devices[device[0]]['weight'] = crush_weight
+                    try:
+                        devices[device[0]] = {}
+                        devices[device[0]]['dev'] = dev
+                        if encrypted:
+                            devices[device[0]]['dmcrypt'] = 'true'
+                        tline = check_output("ceph -c " + conf_file + " osd tree | awk '{print $1,$2,$3,$4}' | grep -w 'osd." + device[0] + "'", shell=True)
+                        osd = tline.split()
+                        if "osd" not in osd[2]:
+                            crush_class = osd[1]
+                            crush_weight = osd[2]
+                            devices[device[0]]['class'] = crush_class
+                            devices[device[0]]['weight'] = crush_weight
+                        else:
+                            crush_weight = osd[1]
+                            devices[device[0]]['weight'] = crush_weight
+                    except CalledProcessError:
+                        continue
                 grain["ceph"]["ceph_disk"] = devices
 
         # keyrings
diff --git a/ceph/backup.sls b/ceph/backup.sls
index e08a679..e590490 100644
--- a/ceph/backup.sls
+++ b/ceph/backup.sls
@@ -40,14 +40,14 @@
   - name: /usr/local/bin/ceph-backup-runner-call.sh
   - user: root
 {%- if backup.client.backup_times is defined %}
-{%- if backup.client.backup_times.dayOfWeek is defined %}
-  - dayweek: {{ backup.client.backup_times.dayOfWeek }}
+{%- if backup.client.backup_times.day_of_week is defined %}
+  - dayweek: {{ backup.client.backup_times.day_of_week }}
 {%- endif -%}
 {%- if backup.client.backup_times.month is defined %}
   - month: {{ backup.client.backup_times.month }}
 {%- endif %}
-{%- if backup.client.backup_times.dayOfMonth is defined %}
-  - daymonth: {{ backup.client.backup_times.dayOfMonth }}
+{%- if backup.client.backup_times.day_of_month is defined %}
+  - daymonth: {{ backup.client.backup_times.day_of_month }}
 {%- endif %}
 {%- if backup.client.backup_times.hour is defined %}
   - hour: {{ backup.client.backup_times.hour }}
@@ -56,7 +56,7 @@
   - minute: {{ backup.client.backup_times.minute }}
 {%- endif %}
 {%- elif backup.client.hours_before_incr is defined %}
-  - minute: '*'
+  - minute: 0
 {%- if backup.client.hours_before_full <= 23 and backup.client.hours_before_full > 1 %}
   - hour: '*/{{ backup.client.hours_before_full }}'
 {%- elif not backup.client.hours_before_full <= 1 %}
@@ -140,14 +140,14 @@
   - name: /usr/local/bin/ceph-backup-runner.sh
   - user: ceph
 {%- if backup.server.backup_times is defined %}
-{%- if backup.server.backup_times.dayOfWeek is defined %}
-  - dayweek: {{ backup.server.backup_times.dayOfWeek }}
+{%- if backup.server.backup_times.day_of_week is defined %}
+  - dayweek: {{ backup.server.backup_times.day_of_week }}
 {%- endif -%}
 {%- if backup.server.backup_times.month is defined %}
   - month: {{ backup.server.backup_times.month }}
 {%- endif %}
-{%- if backup.server.backup_times.dayOfMonth is defined %}
-  - daymonth: {{ backup.server.backup_times.dayOfMonth }}
+{%- if backup.server.backup_times.day_of_month is defined %}
+  - daymonth: {{ backup.server.backup_times.day_of_month }}
 {%- endif %}
 {%- if backup.server.backup_times.hour is defined %}
   - hour: {{ backup.server.backup_times.hour }}
@@ -156,7 +156,7 @@
   - minute: {{ backup.server.backup_times.minute }}
 {%- endif %}
 {%- elif backup.server.hours_before_incr is defined %}
-  - minute: '*'
+  - minute: 0
 {%- if backup.server.hours_before_full <= 23 and backup.server.hours_before_full > 1 %}
   - hour: '*/{{ backup.server.hours_before_full }}'
 {%- elif not backup.server.hours_before_full <= 1 %}
diff --git a/ceph/files/backup/ceph-backup-client-runner-call.sh b/ceph/files/backup/ceph-backup-client-runner-call.sh
index 6c13525..51a0d8d 100644
--- a/ceph/files/backup/ceph-backup-client-runner-call.sh
+++ b/ceph/files/backup/ceph-backup-client-runner-call.sh
@@ -19,7 +19,7 @@
 # Configuration
 # -------------
     BACKUPDIR="{{ backup.backup_dir }}/full"
-    SERVERBACKUPDIR="{{ backup.client.target.get('backup_dir', backup.backup_dir) }}"
+    SERVERBACKUPDIR="{{ backup.client.target.get('backup_dir', backup.backup_dir) }}/full"
     TMPDIR="$( pwd )/tmp_ceph_backup"
     HOSTNAME="$( hostname )"
     TIMESTAMP="$( date +%m%d%k%M )"
@@ -109,4 +109,4 @@
 else
     echo "----------------------------"
     echo "-s parameter passed. Cleanup was not triggered"
-fi
\ No newline at end of file
+fi
diff --git a/ceph/files/jewel/ceph.conf.Debian b/ceph/files/jewel/ceph.conf.Debian
index 02eccdc..a89ebdd 100644
--- a/ceph/files/jewel/ceph.conf.Debian
+++ b/ceph/files/jewel/ceph.conf.Debian
@@ -1,7 +1,7 @@
-{%- from "ceph/map.jinja" import common, mon, osd, radosgw with context %}
+{%- from "ceph/map.jinja" import common, mon, osd, radosgw with context -%}
 [global]
-mon initial members = {%- for member in common.members %}{{ member.name }}{% if not loop.last %},{% endif %}{%- endfor %}
-mon host = {%- for member in common.members %}{{ member.host }}:6789{% if not loop.last %},{% endif %}{%- endfor %}
+mon initial members = {% for member in common.members %}{{ member.name }}{% if not loop.last %},{% endif %}{%- endfor %}
+mon host = {% for member in common.members %}{{ member.host }}:6789{% if not loop.last %},{% endif %}{%- endfor %}
 
 {%- if common.cluster_network is defined %}
 cluster network = {{ common.cluster_network }}
@@ -11,36 +11,25 @@
 {%- endif %}
 
 fsid = {{ common.fsid }}
-
 {%- if common.pg_num is defined %}
 osd pool default pg num = {{ common.pg_num }}
 {%- endif %}
 {%- if common.pgp_num is defined %}
 osd pool default pgp num = {{ common.pgp_num }}
 {%- endif %}
-
-##Global key: value
+{# Global key: value #}
 {%- for key_name, key in common.get('config', {}).get('global', {}).iteritems() %}
-
 {{ key_name }} = {{ key }}
-
 {%- endfor %}
 
-
-##Other sections key: value
+{# Other sections key: value #}
 {%- for key_name, key in common.get('config', {}).iteritems() %}
-
-{%- if key_name not in ['osd', 'mon', 'global'] %}
+{%- if key_name not in ['rgw', 'osd', 'mon', 'global'] %}
 [{{ key_name }}]
-
 {%- for value_name, value in key.iteritems() %}
-
 {{ value_name }} = {{ value }}
-
 {%- endfor %}
-
 {%- endif %}
-
 {%- endfor %}
 
 {%- if osd.bluestore_block_size is defined %}
@@ -108,27 +97,27 @@
 rgw data = /var/lib/ceph/radosgw/{{ common.get('cluster_name', 'ceph') }}-rgw.{{ grains.host }}
 rgw frontends = civetweb port={{ radosgw.bind.address }}:{{ radosgw.bind.port }} num_threads={{ radosgw.threads }}
 rgw dns name = {{ radosgw.get('hostname', grains.host) }}
-
 {%- if radosgw.identity.engine == 'keystone' %}
 {%- set ident = radosgw.identity %}
-
 rgw keystone api version = {{ ident.get('api_version', 3) }}
 rgw keystone url = {{ ident.host }}:{{ ident.get('port', '5000') }}
-rgw keystone accepted roles =  _member_, Member, admin, swiftoperator
-rgw keystone revocation interval = 1000000
-rgw keystone implicit tenants = false
-rgw s3 auth use keystone = true
+rgw keystone accepted roles =  {{ ident.get('accepted_roles', '_member_, Member, admin, swiftoperator') }}
+rgw keystone revocation interval = {{ ident.get('revocation_interval', '1000000') }}
+rgw keystone implicit tenants = {{ ident.get('implicit_tenants', 'false') }}
+rgw s3 auth use keystone = {{ ident.get('s3_auth_use_keystone', 'true') }}
 rgw keystone admin user = {{ ident.get('user', 'admin') }}
 rgw keystone admin password = {{ ident.password }}
-rgw keystone verify ssl = False
+rgw keystone verify ssl = {{ ident.get('keystone_verify_ssl', 'False') }}
 rgw keystone token cache size = {{ ident.get('cache', '10000') }}
 {%- if ident.get('api_version', 3) == 2 %}
 rgw keystone admin tenant = {{ ident.get('tenant', 'admin') }}
 {%- else %}
 rgw keystone admin project = {{ ident.get('project', 'admin') }}
 rgw keystone admin domain = {{ ident.get('domain', 'admin') }}
-{% endif %}
-
 {%- endif %}
-
+{%- endif %}
+{# rgw key: value #}
+{%- for key, value in common.get('config', {}).get('rgw', {}).iteritems() %}
+{{ key }} = {{ value }}
+{%- endfor %}
 {%- endif %}
diff --git a/ceph/files/kraken/ceph.conf.Debian b/ceph/files/kraken/ceph.conf.Debian
index 02eccdc..a89ebdd 100644
--- a/ceph/files/kraken/ceph.conf.Debian
+++ b/ceph/files/kraken/ceph.conf.Debian
@@ -1,7 +1,7 @@
-{%- from "ceph/map.jinja" import common, mon, osd, radosgw with context %}
+{%- from "ceph/map.jinja" import common, mon, osd, radosgw with context -%}
 [global]
-mon initial members = {%- for member in common.members %}{{ member.name }}{% if not loop.last %},{% endif %}{%- endfor %}
-mon host = {%- for member in common.members %}{{ member.host }}:6789{% if not loop.last %},{% endif %}{%- endfor %}
+mon initial members = {% for member in common.members %}{{ member.name }}{% if not loop.last %},{% endif %}{%- endfor %}
+mon host = {% for member in common.members %}{{ member.host }}:6789{% if not loop.last %},{% endif %}{%- endfor %}
 
 {%- if common.cluster_network is defined %}
 cluster network = {{ common.cluster_network }}
@@ -11,36 +11,25 @@
 {%- endif %}
 
 fsid = {{ common.fsid }}
-
 {%- if common.pg_num is defined %}
 osd pool default pg num = {{ common.pg_num }}
 {%- endif %}
 {%- if common.pgp_num is defined %}
 osd pool default pgp num = {{ common.pgp_num }}
 {%- endif %}
-
-##Global key: value
+{# Global key: value #}
 {%- for key_name, key in common.get('config', {}).get('global', {}).iteritems() %}
-
 {{ key_name }} = {{ key }}
-
 {%- endfor %}
 
-
-##Other sections key: value
+{# Other sections key: value #}
 {%- for key_name, key in common.get('config', {}).iteritems() %}
-
-{%- if key_name not in ['osd', 'mon', 'global'] %}
+{%- if key_name not in ['rgw', 'osd', 'mon', 'global'] %}
 [{{ key_name }}]
-
 {%- for value_name, value in key.iteritems() %}
-
 {{ value_name }} = {{ value }}
-
 {%- endfor %}
-
 {%- endif %}
-
 {%- endfor %}
 
 {%- if osd.bluestore_block_size is defined %}
@@ -108,27 +97,27 @@
 rgw data = /var/lib/ceph/radosgw/{{ common.get('cluster_name', 'ceph') }}-rgw.{{ grains.host }}
 rgw frontends = civetweb port={{ radosgw.bind.address }}:{{ radosgw.bind.port }} num_threads={{ radosgw.threads }}
 rgw dns name = {{ radosgw.get('hostname', grains.host) }}
-
 {%- if radosgw.identity.engine == 'keystone' %}
 {%- set ident = radosgw.identity %}
-
 rgw keystone api version = {{ ident.get('api_version', 3) }}
 rgw keystone url = {{ ident.host }}:{{ ident.get('port', '5000') }}
-rgw keystone accepted roles =  _member_, Member, admin, swiftoperator
-rgw keystone revocation interval = 1000000
-rgw keystone implicit tenants = false
-rgw s3 auth use keystone = true
+rgw keystone accepted roles =  {{ ident.get('accepted_roles', '_member_, Member, admin, swiftoperator') }}
+rgw keystone revocation interval = {{ ident.get('revocation_interval', '1000000') }}
+rgw keystone implicit tenants = {{ ident.get('implicit_tenants', 'false') }}
+rgw s3 auth use keystone = {{ ident.get('s3_auth_use_keystone', 'true') }}
 rgw keystone admin user = {{ ident.get('user', 'admin') }}
 rgw keystone admin password = {{ ident.password }}
-rgw keystone verify ssl = False
+rgw keystone verify ssl = {{ ident.get('keystone_verify_ssl', 'False') }}
 rgw keystone token cache size = {{ ident.get('cache', '10000') }}
 {%- if ident.get('api_version', 3) == 2 %}
 rgw keystone admin tenant = {{ ident.get('tenant', 'admin') }}
 {%- else %}
 rgw keystone admin project = {{ ident.get('project', 'admin') }}
 rgw keystone admin domain = {{ ident.get('domain', 'admin') }}
-{% endif %}
-
 {%- endif %}
-
+{%- endif %}
+{# rgw key: value #}
+{%- for key, value in common.get('config', {}).get('rgw', {}).iteritems() %}
+{{ key }} = {{ value }}
+{%- endfor %}
 {%- endif %}
diff --git a/ceph/files/luminous/ceph.conf.Debian b/ceph/files/luminous/ceph.conf.Debian
index 02eccdc..a89ebdd 100644
--- a/ceph/files/luminous/ceph.conf.Debian
+++ b/ceph/files/luminous/ceph.conf.Debian
@@ -1,7 +1,7 @@
-{%- from "ceph/map.jinja" import common, mon, osd, radosgw with context %}
+{%- from "ceph/map.jinja" import common, mon, osd, radosgw with context -%}
 [global]
-mon initial members = {%- for member in common.members %}{{ member.name }}{% if not loop.last %},{% endif %}{%- endfor %}
-mon host = {%- for member in common.members %}{{ member.host }}:6789{% if not loop.last %},{% endif %}{%- endfor %}
+mon initial members = {% for member in common.members %}{{ member.name }}{% if not loop.last %},{% endif %}{%- endfor %}
+mon host = {% for member in common.members %}{{ member.host }}:6789{% if not loop.last %},{% endif %}{%- endfor %}
 
 {%- if common.cluster_network is defined %}
 cluster network = {{ common.cluster_network }}
@@ -11,36 +11,25 @@
 {%- endif %}
 
 fsid = {{ common.fsid }}
-
 {%- if common.pg_num is defined %}
 osd pool default pg num = {{ common.pg_num }}
 {%- endif %}
 {%- if common.pgp_num is defined %}
 osd pool default pgp num = {{ common.pgp_num }}
 {%- endif %}
-
-##Global key: value
+{# Global key: value #}
 {%- for key_name, key in common.get('config', {}).get('global', {}).iteritems() %}
-
 {{ key_name }} = {{ key }}
-
 {%- endfor %}
 
-
-##Other sections key: value
+{# Other sections key: value #}
 {%- for key_name, key in common.get('config', {}).iteritems() %}
-
-{%- if key_name not in ['osd', 'mon', 'global'] %}
+{%- if key_name not in ['rgw', 'osd', 'mon', 'global'] %}
 [{{ key_name }}]
-
 {%- for value_name, value in key.iteritems() %}
-
 {{ value_name }} = {{ value }}
-
 {%- endfor %}
-
 {%- endif %}
-
 {%- endfor %}
 
 {%- if osd.bluestore_block_size is defined %}
@@ -108,27 +97,27 @@
 rgw data = /var/lib/ceph/radosgw/{{ common.get('cluster_name', 'ceph') }}-rgw.{{ grains.host }}
 rgw frontends = civetweb port={{ radosgw.bind.address }}:{{ radosgw.bind.port }} num_threads={{ radosgw.threads }}
 rgw dns name = {{ radosgw.get('hostname', grains.host) }}
-
 {%- if radosgw.identity.engine == 'keystone' %}
 {%- set ident = radosgw.identity %}
-
 rgw keystone api version = {{ ident.get('api_version', 3) }}
 rgw keystone url = {{ ident.host }}:{{ ident.get('port', '5000') }}
-rgw keystone accepted roles =  _member_, Member, admin, swiftoperator
-rgw keystone revocation interval = 1000000
-rgw keystone implicit tenants = false
-rgw s3 auth use keystone = true
+rgw keystone accepted roles =  {{ ident.get('accepted_roles', '_member_, Member, admin, swiftoperator') }}
+rgw keystone revocation interval = {{ ident.get('revocation_interval', '1000000') }}
+rgw keystone implicit tenants = {{ ident.get('implicit_tenants', 'false') }}
+rgw s3 auth use keystone = {{ ident.get('s3_auth_use_keystone', 'true') }}
 rgw keystone admin user = {{ ident.get('user', 'admin') }}
 rgw keystone admin password = {{ ident.password }}
-rgw keystone verify ssl = False
+rgw keystone verify ssl = {{ ident.get('keystone_verify_ssl', 'False') }}
 rgw keystone token cache size = {{ ident.get('cache', '10000') }}
 {%- if ident.get('api_version', 3) == 2 %}
 rgw keystone admin tenant = {{ ident.get('tenant', 'admin') }}
 {%- else %}
 rgw keystone admin project = {{ ident.get('project', 'admin') }}
 rgw keystone admin domain = {{ ident.get('domain', 'admin') }}
-{% endif %}
-
 {%- endif %}
-
+{%- endif %}
+{# rgw key: value #}
+{%- for key, value in common.get('config', {}).get('rgw', {}).iteritems() %}
+{{ key }} = {{ value }}
+{%- endfor %}
 {%- endif %}