Keystone fernet key rotation without gluster
In order to get rid of keystone fernet key directory being
managed by a shared filesystem, this patch contains salt orchestrate states
to create needed directories, generate ssh keys and put public keys
to respective nodes PRIOR to keystone installation. Rsync script
is used to copy fernet keys from primary control node to all the rest of
control nodes.
Change-Id: I6a11870a59301902cf1bc12624c1fd86d8e816b8
Related-PROD: PROD-19972
diff --git a/README.rst b/README.rst
index bc98bf2..a171ab2 100644
--- a/README.rst
+++ b/README.rst
@@ -811,6 +811,40 @@
#. Apply the :command:`keystone.client` state.
+Fernet-keys rotation without gluster
+------------------------------------
+
+In the future fernet keys supposed to be rotated with rsync+ssh instead of using glusterfs. By default it is assumed
+that the script will run on primary control node (ctl01) and will rotate and transfer fernet keys to secondary
+controller nodes (ctl02, ctl03). Following parameter should be set on cluster level:
+
+keystone_node_role
+
+and fernet_rotation_driver should be set to 'rsync'
+
+By default this parameter is set to "secondary" on system level along with other parameters:
+.. code-block:: yaml
+
+ keystone:
+ server:
+ role: ${_param:keystone_node_role}
+ tokens:
+ fernet_sync_nodes_list:
+ control02:
+ name: ctl02
+ enabled: True
+ control03:
+ name: ctl03
+ enabled: True
+ fernet_rotation_driver: rsync
+
+Prior to running keystone salt states ssh key should be generated and its public part should be placed on secondary controllers.
+It can be accomplished by running following orchestration state before keystone states:
+
+salt-run state.orchestrate keystone.orchestrate.deploy
+
+Currently the default fernet rotation driver is a shared filesystem
+
Documentation and Bugs
======================
diff --git a/keystone/files/fernet_keys_rotate.sh b/keystone/files/fernet_keys_rotate.sh
new file mode 100644
index 0000000..636e315
--- /dev/null
+++ b/keystone/files/fernet_keys_rotate.sh
@@ -0,0 +1,100 @@
+{%- from "keystone/map.jinja" import server with context -%}
+#!/bin/bash
+usage() {
+cat <<EOF
+Script for Fernet key rotation and sync
+ For additional help please use: $0 -h or --help
+ example: $0 -s
+EOF
+}
+
+if [ $# -lt "1" ]; then
+ usage
+ exit 1
+fi
+
+help_usage() {
+cat <<EOF
+Following options are supported:
+ -s Perform sync to secondary controller nodes only
+ -r Perform Fernet key rotation on primary controller only
+ -rs Perform Fernet key rotation on primary controller and sync to secondary controller nodes
+EOF
+}
+
+if [ $# -lt "1" ]; then
+ usage
+ exit 1
+fi
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+ -h|--help)
+ help_usage
+ shift # past argument
+ ;;
+ -s)
+ MODE="SYNC"
+ shift # past argument
+ ;;
+ -r)
+ MODE="ROTATE"
+ shift
+ ;;
+ -rs)
+ MODE="ROTATE_AND_SYNC"
+ shift
+ ;;
+ *) # unknown option
+ echo "Unknown option. Please refer to help section by passing -h or --help option"
+ shift # past argument
+ ;;
+esac
+done
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+#Setting variables
+KEYSTONE_MANAGE_CMD="/usr/bin/keystone-manage"
+{%- if server.tokens.fernet_sync_nodes_list is defined %}
+ {%- set _nodes = [] %}
+ {%- for node_name, fernet_sync_nodes_list in server.tokens.get('fernet_sync_nodes_list', {}).iteritems() %}
+ {%- if fernet_sync_nodes_list.get('enabled', False) %}
+ {%- do _nodes.append(fernet_sync_nodes_list.name) %}
+ {%- endif %}
+ {%- endfor %}
+NODES="{{ ' '.join(_nodes) }}"
+{%- else %}
+NODES=""
+{%- endif %}
+
+if [[ ${MODE} == 'SYNC' ]]; then
+ echo "Running in SYNC mode"
+ if [[ ${NODES} != '' ]]; then
+ for NODE in ${NODES}; do
+ echo "${NODE}"
+ rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --delete {{ server.tokens.location }}/ keystone@${NODE}:{{ server.tokens.location }}/
+ done
+ else
+ echo "List of nodes is not specified, no need for sync, exiting"
+ exit 0
+ fi
+elif [[ ${MODE} == 'ROTATE' ]]; then
+ echo "Running in ROTATE mode"
+ /usr/bin/keystone-manage --log-file /var/log/keystone/keystone-rotate.log fernet_rotate --keystone-user keystone --keystone-group keystone
+elif [[ ${MODE} == 'ROTATE_AND_SYNC' ]]; then
+ echo "Running in ROTATE_AND_SYNC mode"
+ /usr/bin/keystone-manage --log-file /var/log/keystone/keystone-rotate.log fernet_rotate --keystone-user keystone --keystone-group keystone
+ if [[ ${NODES} != '' ]]; then
+ for NODE in ${NODES}; do
+ echo "${NODE}"
+ rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --delete {{ server.tokens.location }}/ keystone@${NODE}:{{ server.tokens.location }}/
+ done
+ else
+ echo "List of nodes is not specified, no need for sync, exiting"
+ exit 0
+ fi
+fi
diff --git a/keystone/meta/salt.yml b/keystone/meta/salt.yml
index 92324e5..554ed23 100644
--- a/keystone/meta/salt.yml
+++ b/keystone/meta/salt.yml
@@ -7,6 +7,12 @@
control:
priority: 520
+orchestration:
+ deploy:
+ applications:
+ keystone:
+ priority: 1000
+
minion:
{%- if pillar.get('keystone', {}).get('server') or pillar.get('keystone', {}).get('client') %}
{%- from "keystone/map.jinja" import server with context %}
diff --git a/keystone/orchestrate/deploy.sls b/keystone/orchestrate/deploy.sls
new file mode 100644
index 0000000..eccbbab
--- /dev/null
+++ b/keystone/orchestrate/deploy.sls
@@ -0,0 +1,36 @@
+{%- set minions = ((salt['cmd.shell']("salt '*' match.pillar 'keystone:server:role:primary' --out=json --static"))|load_json).values() %}
+keystone_ssh_keys:
+ salt.state:
+ - tgt: 'I@keystone:server:role:primary'
+ - tgt_type: compound
+ - sls: keystone.orchestrate.generate_keystone_ssh_keys
+ - onlyif: {% if True in minions %}/bin/true{% else %}/bin/false{% endif %}
+
+send_keystone_public_key:
+ salt.state:
+ - tgt: 'I@keystone:server:role:primary'
+ - tgt_type: compound
+ - sls: keystone.orchestrate.send_keystone_public_key
+ - require:
+ - salt: keystone_ssh_keys
+ - onlyif: {% if True in minions %}/bin/true{% else %}/bin/false{% endif %}
+
+{%- set minions = ((salt['cmd.shell']("salt '*' match.pillar 'keystone:server' --out=json --static"))|load_json).values() %}
+salt_mine_update:
+ salt.state:
+ - tgt: 'I@keystone:server'
+ - tgt_type: compound
+ - sls: keystone.orchestrate.run_mine_update
+ - require:
+ - salt: send_keystone_public_key
+ - onlyif: {% if True in minions %}/bin/true{% else %}/bin/false{% endif %}
+
+{%- set minions = ((salt['cmd.shell']("salt '*' match.pillar 'keystone:server:role:secondary' --out=json --static"))|load_json).values() %}
+get_keystone_public_key:
+ salt.state:
+ - tgt: 'I@keystone:server:role:secondary'
+ - tgt_type: compound
+ - sls: keystone.orchestrate.get_keystone_public_key
+ - require:
+ - salt: salt_mine_update
+ - onlyif: {% if True in minions %}/bin/true{% else %}/bin/false{% endif %}
diff --git a/keystone/orchestrate/generate_keystone_ssh_keys.sls b/keystone/orchestrate/generate_keystone_ssh_keys.sls
new file mode 100644
index 0000000..c5f7ccb
--- /dev/null
+++ b/keystone/orchestrate/generate_keystone_ssh_keys.sls
@@ -0,0 +1,19 @@
+{%- from "keystone/map.jinja" import server with context %}
+applying_generate_keystone_ssh_keys_state:
+ test.succeed_without_changes
+{% if server.tokens.get('fernet_rotation_driver', 'shared_filesystem') == 'rsync' %}
+keystone_fernet_keys:
+ file.directory:
+ - name: {{ server.tokens.location }}
+ - mode: 750
+ - user: keystone
+ - group: keystone
+
+generate_keystone_ssh_keys:
+ cmd.run:
+ - name: ssh-keygen -q -N '' -f /var/lib/keystone/.ssh/id_rsa
+ - runas: keystone
+ - creates: /var/lib/keystone/.ssh/id_rsa
+ - require:
+ - file: {{ server.tokens.location }}
+{%- endif %}
diff --git a/keystone/orchestrate/get_keystone_public_key.sls b/keystone/orchestrate/get_keystone_public_key.sls
new file mode 100644
index 0000000..2be74e1
--- /dev/null
+++ b/keystone/orchestrate/get_keystone_public_key.sls
@@ -0,0 +1,37 @@
+{%- from "keystone/map.jinja" import server with context %}
+applying_get_keystone_public_key_state:
+ test.succeed_without_changes
+{% if server.tokens.get('fernet_rotation_driver', 'shared_filesystem') == 'rsync' %}
+{%- set authorized_keys = salt['mine.get']('I@keystone:server:role:primary', 'keystone_public_key', 'compound') %}
+
+keystone_fernet_keys:
+ file.directory:
+ - name: {{ server.tokens.location }}
+ - mode: 650
+ - user: keystone
+ - group: keystone
+
+/var/lib/keystone/.ssh:
+ file.directory:
+ - user: keystone
+ - group: keystone
+ - file_mode: 600
+ - dir_mode: 600
+ - makedirs: True
+ - recurse:
+ - user
+ - group
+ - mode
+
+ {% if authorized_keys is defined and authorized_keys|length > 0 %}
+put_keystone_file:
+ file.managed:
+ - name: /var/lib/keystone/.ssh/authorized_keys
+ - contents: '{{ authorized_keys.values()[0] }}'
+ - user: keystone
+ - group: keystone
+ - mode: 600
+ - require:
+ - file: /var/lib/keystone/.ssh
+ {%- endif %}
+{%- endif %}
diff --git a/keystone/orchestrate/run_mine_update.sls b/keystone/orchestrate/run_mine_update.sls
new file mode 100644
index 0000000..33e626c
--- /dev/null
+++ b/keystone/orchestrate/run_mine_update.sls
@@ -0,0 +1,8 @@
+{%- from "keystone/map.jinja" import server with context %}
+applying_run_mine_update_state:
+ test.succeed_without_changes
+{% if server.tokens.get('fernet_rotation_driver', 'shared_filesystem') == 'rsync' %}
+execute_mine_update:
+ module.run:
+ - name: mine.update
+{%- endif %}
diff --git a/keystone/orchestrate/send_keystone_public_key.sls b/keystone/orchestrate/send_keystone_public_key.sls
new file mode 100644
index 0000000..7edaf14
--- /dev/null
+++ b/keystone/orchestrate/send_keystone_public_key.sls
@@ -0,0 +1,13 @@
+{%- from "keystone/map.jinja" import server with context %}
+applying_send_keystone_public_key_state:
+ test.succeed_without_changes
+{% if server.tokens.get('fernet_rotation_driver', 'shared_filesystem') == 'rsync' %}
+mine_send_keystone_public_key:
+ module.run:
+ - name: mine.send
+ - func: keystone_public_key
+ - args:
+ - 'cat /var/lib/keystone/.ssh/id_rsa.pub'
+ - kwargs:
+ mine_function: cmd.run
+{%- endif %}
diff --git a/keystone/server.sls b/keystone/server.sls
index 9b43303..6e236c4 100644
--- a/keystone/server.sls
+++ b/keystone/server.sls
@@ -331,6 +331,31 @@
- onlyif: /bin/false
{%- endif %}
+ {% if server.tokens.get('fernet_rotation_driver', 'shared_filesystem') == 'rsync' %}
+ {% if server.get('role', 'secondary') == 'primary' %}
+/var/lib/keystone/fernet_keys_rotate.sh:
+ file.managed:
+ - source: salt://keystone/files/fernet_keys_rotate.sh
+ - template: jinja
+ - user: keystone
+ - group: keystone
+ - mode: 744
+ - require:
+ - pkg: keystone_packages
+ - file: keystone_fernet_keys
+ - cmd: keystone_fernet_setup
+
+run_fernet_rotation_script_in_sync_mode:
+ cmd.run:
+ - name: /var/lib/keystone/fernet_keys_rotate.sh -s
+ - runas: keystone
+ - require:
+ - pkg: keystone_packages
+ - file: /var/lib/keystone/fernet_keys_rotate.sh
+
+ {%- endif %}
+ {%- endif %}
+
{% endif %}
{%- if server.version in ['newton', 'ocata', 'pike'] %}