Orchestration state (and related stuff) to update GlusterFS
Prod-Related: PROD-29243
Change-Id: I8ec489b5772f48aae8d6d9a63f4ec160ad61b78f
diff --git a/README.rst b/README.rst
index 4961b2b..cb81725 100644
--- a/README.rst
+++ b/README.rst
@@ -19,6 +19,18 @@
* ``glusterfs.client``
Sets up GlusterFS client
+* ``glusterfs.update.server``
+ Update GlusterFS on servers
+
+* ``glusterfs.update.client``
+ Update GlusterFS on clients
+
+* ``glusterfs.update.op_version``
+ Update GlusterFS cluster.op-version option
+
+* ``glusterfs.orchestrate.update``
+ Orchestrate an update of GlusterFS process
+
Available metadata
==================
diff --git a/glusterfs/files/validate_glusterfs_update.sh b/glusterfs/files/validate_glusterfs_update.sh
new file mode 100644
index 0000000..2a21a1b
--- /dev/null
+++ b/glusterfs/files/validate_glusterfs_update.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#!/bin/bash
+
+DESIRED_VERSION=${1:-5}
+if [[ ${2:-none} == 'client' ]]; then
+ IS_CLIENT=true
+else
+ IS_CLIENT=false
+fi
+
+## Check version of binary
+GLUSTERFS_VERSION=$(glusterfsd --version | head -n1 | awk '{print $2}')
+[[ $GLUSTERFS_VERSION =~ ^$DESIRED_VERSION ]] || {
+ echo
+ echo "changed=yes comment='incorrect version $GLUSTERFS_VERSION'"
+ exit 1
+}
+
+## The rest of checks are relevant only for server
+$IS_CLIENT && exit 0
+
+which gluster >/dev/null || {
+ echo
+ echo "changed=yes comment='gluster command does not exist'"
+ exit 1
+}
+
+for vol in $(gluster volume list); do
+ ## Check volumes' status
+ gluster volume info $vol | grep -q 'Status: Started' || {
+ echo
+ echo "changed=yes comment='volume $vol is not started'"
+ exit 1;
+ }
+ ## Check volumes' heal info
+ NUMBER_OF_BRICKS=$(gluster volume info $vol | grep -cE 'Brick[0-9]+:')
+ BRICKS_CONNECTED=$(gluster volume heal $vol info | grep -c 'Status: Connected')
+ [[ $NUMBER_OF_BRICKS -ne $BRICKS_CONNECTED ]] && {
+ echo
+ echo "changed=yes comment='some bricks of volume $vol is not connected'"
+ exit 1;
+ }
+ BRICKS_ONLINE=$(gluster volume status $vol detail | awk -F: '/^Online/ {print $2}' | fgrep -c Y)
+ [[ $NUMBER_OF_BRICKS -ne $BRICKS_ONLINE ]] && {
+ echo
+ echo "changed=yes comment='some bricks of volume $vol is not online'"
+ exit 1;
+ }
+
+done
+
+exit 0
diff --git a/glusterfs/orchestrate/update.sls b/glusterfs/orchestrate/update.sls
new file mode 100644
index 0000000..cfad9e1
--- /dev/null
+++ b/glusterfs/orchestrate/update.sls
@@ -0,0 +1,53 @@
+{# Update repo with new packages #}
+glusterfs.update.repo:
+ salt.state:
+ - tgt: 'I@glusterfs:server or I@glusterfs:client'
+ - tgt_type: compound
+ - sls: linux.system.repo
+
+{# Update servers one-by-one #}
+glusterfs.update.server:
+ salt.state:
+ - tgt: 'glusterfs:server'
+ - tgt_type: pillar
+ - sls: glusterfs.update.server
+ - batch: 1
+ - require:
+ - salt: glusterfs.update.repo
+
+{# Update clients one-by-one #}
+glusterfs.update.client:
+ salt.state:
+ - tgt: 'glusterfs:client'
+ - tgt_type: pillar
+ - sls: glusterfs.update.client
+ - batch: 1
+ - require:
+ - salt: glusterfs.update.repo
+ - salt: glusterfs.update.server
+
+{# Update cluster.op-version #}
+glusterfs.update.op-version:
+ salt.state:
+ - tgt: 'glusterfs:server:role:primary'
+ - tgt_type: pillar
+ - sls: glusterfs.update.op_version
+ - require:
+ - salt: glusterfs.update.server
+ - salt: glusterfs.update.client
+
+glusterfs.server.service:
+ salt.state:
+ - tgt: 'glusterfs:server'
+ - tgt_type: pillar
+ - sls: glusterfs.server.service
+ - onfail:
+ - salt: glusterfs.update.server
+
+glusterfs.client:
+ salt.state:
+ - tgt: 'glusterfs:client'
+ - tgt_type: pillar
+ - sls: glusterfs.client
+ - onfail:
+ - salt: glusterfs.update.client
diff --git a/glusterfs/update/client.sls b/glusterfs/update/client.sls
new file mode 100644
index 0000000..a40417d
--- /dev/null
+++ b/glusterfs/update/client.sls
@@ -0,0 +1,60 @@
+{% from "glusterfs/map.jinja" import client with context %}
+
+{# Ensure newest package is available #}
+{% set latest_pkg_version = salt['pkg.latest_version']('glusterfs-client') %}
+{% set desired_version = salt['pillar.get']('_param:linux_system_repo_mcp_glusterfs_version_number') %}
+{% if latest_pkg_version and salt['pkg.version_cmp'](latest_pkg_version, desired_version) >= 0 %}
+ {% set ready_to_upgrade = True %}
+{% else %}
+ {% set ready_to_upgrade = False %}
+{% endif %}
+
+
+{%- if client.enabled and ready_to_upgrade %}
+
+{# TODO: support kdt #}
+{# Drain #}
+drain_docker_payload:
+ cmd.run:
+ - name: docker node update --availability drain {{ grains.nodename }}
+ - onlyif: which docker
+
+{# Update #}
+glusterfs_kill_processes:
+ process.absent:
+ - name: /usr/sbin/gluster
+
+glusterfs_install_latest_packages:
+ pkg.latest:
+ - pkgs: {{ client.pkgs }}
+ - refresh: true
+ - require:
+ - process: glusterfs_kill_processes
+
+{# Validate #}
+glusterfs_validate_client_update:
+ cmd.script:
+ - name: validate_glusterfs_update.sh {{ desired_version }} client
+ - source: salt://glusterfs/files/validate_glusterfs_update.sh
+ - stateful: True
+ - require:
+ - pkg: glusterfs_install_latest_packages
+ - retry: True
+
+{# run glusterfs.client to restart mounts #}
+glusterfs_apply_client_state:
+ module.run:
+ - name: state.sls
+ - mods: glusterfs.client
+ - require:
+ - cmd: glusterfs_validate_client_update
+
+{# TODO: support kdt #}
+{# (Un)Drain #}
+restore_docker_node_availability:
+ cmd.run:
+ - name: docker node update --availability active {{ grains.nodename }}
+ - onlyif: which docker
+ - order: last
+
+{%- endif %}
diff --git a/glusterfs/update/op_version.sls b/glusterfs/update/op_version.sls
new file mode 100644
index 0000000..d50e9de
--- /dev/null
+++ b/glusterfs/update/op_version.sls
@@ -0,0 +1,15 @@
+{# get cluster.max-op-version #}
+
+{% set max_op_version = salt['cmd.shell']("gluster volume get all cluster.max-op-version 2>/dev/null | awk '/max-op-version/ {print $2}'") %}
+{% set current_op_version = salt['cmd.shell']("gluster volume get all cluster.op-version 2>/dev/null | awk '/op-version/ {print $2}'") %}
+
+{# set cluster.op-version #}
+{% if max_op_version and current_op_version and max_op_version != current_op_version %}
+glusterfs_set_cluster_op_version:
+ cmd.run:
+ - name: gluster volume set all cluster.op-version {{ max_op_version }}
+ - unless: "gluster volume get all cluster.op-version 2>/dev/null | grep -q {{ max_op_version }}"
+{% else %}
+glusterfs_no_update_op_version:
+ test.nop
+{% endif %}
diff --git a/glusterfs/update/server.sls b/glusterfs/update/server.sls
new file mode 100644
index 0000000..7859ec8
--- /dev/null
+++ b/glusterfs/update/server.sls
@@ -0,0 +1,82 @@
+{% from "glusterfs/map.jinja" import server with context %}
+
+{# Ensure newest package is available #}
+{% set latest_pkg_version = salt['pkg.latest_version']('glusterfs-server') %}
+{% set desired_version = salt['pillar.get']('_param:linux_system_repo_mcp_glusterfs_version_number') %}
+{% if latest_pkg_version and salt['pkg.version_cmp'](latest_pkg_version, desired_version) >= 0 %}
+ {% set ready_to_upgrade = True %}
+{% else %}
+ {% set ready_to_upgrade = False %}
+{% endif %}
+
+
+{%- if server.enabled and ready_to_upgrade %}
+
+{# reset some options from volumes which are not supported by glusterfs 5+ #}
+ {%- if server.volumes is defined %}
+ {%- for name, volume in server.volumes.iteritems() %}
+glusterfs_reset_lock_heal_option_{{ name }}:
+ cmd.run:
+ - name: gluster volume reset {{ name }} features.lock-heal
+ - onlyif: "gluster volume info {{ name }} | grep -q features.lock-heal"
+ - require_in:
+ - service: glusterfs_stop_service
+
+glusterfs_reset_grace_timeout_option_{{ name }}:
+ cmd.run:
+ - name: gluster volume reset {{ name }} features.grace-timeout
+ - onlyif: "gluster volume info {{ name }} | grep -q features.grace-timeout"
+ - require_in:
+ - service: glusterfs_stop_service
+ {%- endfor %}
+ {%- endif %}
+
+{# Update #}
+glusterfs_stop_service:
+ service.dead:
+ - name: /usr/sbin/gluster
+
+glusterfs_kill_processes:
+ process.absent:
+ - name: gluster
+ - require:
+ - service: glusterfs_stop_service
+
+glusterfs_install_latest_packages:
+ pkg.latest:
+ - pkgs: {{ server.pkgs }}
+ - refresh: true
+ - require:
+ - service: glusterfs_stop_service
+ - process: glusterfs_kill_processes
+
+glusterfs_service_running:
+ service.running:
+ - name: {{ server.service }}
+ - enable: true
+ - require:
+ - pkg: glusterfs_install_latest_packages
+
+{# heal volumes, just in case #}
+ {%- if server.volumes is defined %}
+ {%- for name, volume in server.volumes.iteritems() %}
+glusterfs_volume_heal_{{ name }}:
+ cmd.run:
+ - name: gluster volume heal {{ name }}
+ - require:
+ - service: glusterfs_service_running
+ - require_in:
+ - cmd: glusterfs_validate_server_update
+ - retry: True
+ {%- endfor %}
+ {%- endif %}
+
+{# Validate #}
+glusterfs_validate_server_update:
+ cmd.script:
+ - name: validate_glusterfs_update.sh {{ desired_version }}
+ - source: salt://glusterfs/files/validate_glusterfs_update.sh
+ - stateful: True
+ - retry: True
+
+{%- endif %}