Merge "Update run_tests.sh to the latest revision"
diff --git a/README.rst b/README.rst
index 9b2f7fe..ef4d3de 100644
--- a/README.rst
+++ b/README.rst
@@ -34,6 +34,87 @@
- host: ${_param:cluster_node03_address}
id: 3
+Backup client with ssh/rsync remote host
+
+.. code-block:: yaml
+
+ zookeeper:
+ backup:
+ client:
+ enabled: true
+ full_backups_to_keep: 3
+ hours_before_full: 24
+ target:
+ host: cfg01
+
+ .. note:: full_backups_to_keep param states how many backup will be stored locally on zookeeper client.
+ More options to relocate local backups can be done using salt-formula-backupninja.
+
+Backup client with local backup only
+
+.. code-block:: yaml
+
+ zookeeper:
+ backup:
+ client:
+ enabled: true
+ full_backups_to_keep: 3
+ hours_before_full: 24
+
+ .. note:: full_backups_to_keep param states how many backup will be stored locally on zookeeper client
+
+
+Backup server rsync
+
+.. code-block:: yaml
+
+ zookeeper:
+ backup:
+ server:
+ enabled: true
+ hours_before_full: 24
+ full_backups_to_keep: 5
+ key:
+ zookeeper_pub_key:
+ enabled: true
+ key: ssh_rsa
+
+Client restore from local backup:
+
+.. code-block:: yaml
+
+ zookeeper:
+ backup:
+ client:
+ enabled: true
+ full_backups_to_keep: 3
+ hours_before_full: 24
+ target:
+ host: cfg01
+ restore_latest: 1
+ restore_from: local
+
+ .. note:: restore_latest param with a value of 1 means to restore db from the last full backup. 2 would mean to restore second latest full backup.
+
+
+Client restore from remote backup:
+
+.. code-block:: yaml
+
+ zookeeper:
+ backup:
+ client:
+ enabled: true
+ full_backups_to_keep: 3
+ hours_before_full: 24
+ target:
+ host: cfg01
+ restore_latest: 1
+ restore_from: remote
+
+ .. note:: restore_latest param with a value of 1 means to restore db from the last full backup. 2 would mean to restore second latest full backup.
+
+
Read more
=========
diff --git a/tests/pillar/backup_client.sls b/tests/pillar/backup_client.sls
new file mode 100644
index 0000000..d7c2f52
--- /dev/null
+++ b/tests/pillar/backup_client.sls
@@ -0,0 +1,10 @@
+zookeeper:
+ backup:
+ client:
+ enabled: true
+ full_backups_to_keep: 3
+ hours_before_full: 24
+ target:
+ host: cfg01
+ restore_latest: 1
+ restore_from: local
\ No newline at end of file
diff --git a/tests/pillar/backup_server.sls b/tests/pillar/backup_server.sls
new file mode 100644
index 0000000..04b4bd7
--- /dev/null
+++ b/tests/pillar/backup_server.sls
@@ -0,0 +1,10 @@
+zookeeper:
+ backup:
+ server:
+ enabled: true
+ hours_before_full: 24
+ full_backups_to_keep: 5
+ key:
+ zookeeper_pub_key:
+ enabled: true
+ key: ssh-rsa
\ No newline at end of file
diff --git a/zookeeper/backup.sls b/zookeeper/backup.sls
new file mode 100644
index 0000000..05c29ee
--- /dev/null
+++ b/zookeeper/backup.sls
@@ -0,0 +1,151 @@
+{%- from "zookeeper/map.jinja" import backup with context %}
+
+{%- if backup.client is defined %}
+
+{%- if backup.client.enabled %}
+
+zookeeper_backup_client_packages:
+ pkg.installed:
+ - names: {{ backup.pkgs }}
+
+zookeeper_backup_runner_script:
+ file.managed:
+ - name: /usr/local/bin/zookeeper-backup-runner.sh
+ - source: salt://zookeeper/files/backup/zookeeper-backup-client-runner.sh
+ - template: jinja
+ - mode: 655
+ - require:
+ - pkg: zookeeper_backup_client_packages
+
+zookeeper_backup_dir:
+ file.directory:
+ - name: {{ backup.backup_dir }}/full
+ - user: root
+ - group: root
+ - makedirs: true
+
+zookeeper_backup_runner_cron:
+ cron.present:
+ - name: /usr/local/bin/zookeeper-backup-runner.sh
+ - user: root
+{%- if not backup.cron %}
+ - commented: True
+{%- endif %}
+ - minute: 0
+{%- if backup.client.hours_before_full is defined %}
+{%- if backup.client.hours_before_full <= 23 and backup.client.hours_before_full > 1 %}
+ - hour: '*/{{ backup.client.hours_before_full }}'
+{%- elif not backup.client.hours_before_full <= 1 %}
+ - hour: 2
+{%- endif %}
+{%- else %}
+ - hour: 2
+{%- endif %}
+ - require:
+ - file: zookeeper_backup_runner_script
+
+{%- if backup.client.restore_latest is defined %}
+
+zookeeper_backup_restore_script:
+ file.managed:
+ - name: /usr/local/bin/zookeeper-backup-restore.sh
+ - source: salt://zookeeper/files/backup/zookeeper-backup-client-restore.sh
+ - template: jinja
+ - mode: 655
+ - require:
+ - pkg: zookeeper_backup_client_packages
+
+zookeeper_backup_call_restore_script:
+ file.managed:
+ - name: /usr/local/bin/zookeeper-backup-restore-call.sh
+ - source: salt://zookeeper/files/backup/zookeeper-backup-client-restore-call.sh
+ - template: jinja
+ - mode: 655
+ - require:
+ - file: zookeeper_backup_restore_script
+
+zookeeper_run_restore:
+ cmd.run:
+ - name: /usr/local/bin/zookeeper-backup-restore-call.sh
+ - unless: "[ -e {{ backup.backup_dir }}/dbrestored ]"
+ - require:
+ - file: zookeeper_backup_call_restore_script
+
+{%- endif %}
+
+{%- endif %}
+
+{%- endif %}
+
+{%- if backup.server is defined %}
+
+{%- if backup.server.enabled %}
+
+zookeeper_backup_server_packages:
+ pkg.installed:
+ - names: {{ backup.pkgs }}
+
+zookeeper_user:
+ user.present:
+ - name: zookeeper
+ - system: true
+ - home: {{ backup.backup_dir }}
+
+{{ backup.backup_dir }}/full:
+ file.directory:
+ - mode: 755
+ - user: zookeeper
+ - group: zookeeper
+ - makedirs: true
+ - require:
+ - user: zookeeper_user
+ - pkg: zookeeper_backup_server_packages
+
+{%- for key_name, key in backup.server.key.iteritems() %}
+
+{%- if key.get('enabled', False) %}
+
+zookeeper_key_{{ key.key }}:
+ ssh_auth.present:
+ - user: zookeeper
+ - name: {{ key.key }}
+ - require:
+ - file: {{ backup.backup_dir }}/full
+
+{%- endif %}
+
+{%- endfor %}
+
+zookeeper_server_script:
+ file.managed:
+ - name: /usr/local/bin/zookeeper-backup-runner.sh
+ - source: salt://zookeeper/files/backup/zookeeper-backup-server-runner.sh
+ - template: jinja
+ - mode: 655
+ - require:
+ - pkg: zookeeper_backup_server_packages
+
+zookeeper_server_cron:
+ cron.present:
+ - name: /usr/local/bin/zookeeper-backup-runner.sh
+ - user: zookeeper
+{%- if not backup.cron %}
+ - commented: True
+{%- endif %}
+ - minute: 0
+ - hour: 2
+ - require:
+ - file: zookeeper_server_script
+
+zookeeper_server_call_restore_script:
+ file.managed:
+ - name: /usr/local/bin/zookeeper-restore-call.sh
+ - source: salt://zookeeper/files/backup/zookeeper-backup-server-restore-call.sh
+ - template: jinja
+ - mode: 655
+ - require:
+ - pkg: zookeeper_backup_server_packages
+
+{%- endif %}
+
+{%- endif %}
diff --git a/zookeeper/files/backup/zookeeper-backup-client-restore-call.sh b/zookeeper/files/backup/zookeeper-backup-client-restore-call.sh
new file mode 100644
index 0000000..206cff6
--- /dev/null
+++ b/zookeeper/files/backup/zookeeper-backup-client-restore-call.sh
@@ -0,0 +1,62 @@
+{%- from "zookeeper/map.jinja" import backup with context %}
+#!/bin/bash
+
+# Script is to locally prepare appropriate backup to restore from local or remote location and call client-restore script in for loop with every keyspace
+
+# Configuration
+# -------------
+ BACKUPDIR="{{ backup.backup_dir }}/full"
+ SCRIPTDIR="/usr/local/bin"
+ DBALREADYRESTORED="{{ backup.backup_dir }}/dbrestored"
+ LOGDIR="/var/log/backups"
+ LOGFILE="/var/log/backups/zookeeper-restore.log"
+ SCPLOG="/var/log/backups/zookeeper-restore-scp.log"
+
+
+if [ -e $DBALREADYRESTORED ]; then
+ error "Databases already restored. If you want to restore again delete $DBALREADYRESTORED file and run the script again."
+fi
+
+# Create backup directory.
+if [ ! -d "$LOGDIR" ] && [ ! -e "$LOGDIR" ]; then
+ mkdir -p "$LOGDIR"
+fi
+
+{%- if backup.client.restore_from == 'remote' %}
+
+echo "Adding ssh-key of remote host to known_hosts"
+ssh-keygen -R {{ backup.client.target.host }} 2>&1 | > $SCPLOG
+ssh-keyscan {{ backup.client.target.host }} >> ~/.ssh/known_hosts 2>&1 | >> $SCPLOG
+REMOTEBACKUPPATH=`ssh zookeeper@{{ backup.client.target.host }} "/usr/local/bin/zookeeper-restore-call.sh {{ backup.client.restore_latest }}"`
+
+#get files from remote and change variables to local restore dir
+
+LOCALRESTOREDIR=/var/backups/restoreZookeeper
+FULLBACKUPDIR=$LOCALRESTOREDIR/full
+
+mkdir -p $LOCALRESTOREDIR
+rm -rf $LOCALRESTOREDIR/*
+
+echo "SCP getting full backup files"
+FULL=`basename $REMOTEBACKUPPATH`
+mkdir -p $FULLBACKUPDIR
+`scp -rp zookeeper@{{ backup.client.target.host }}:$REMOTEBACKUPPATH $FULLBACKUPDIR/$FULL/ >> $SCPLOG 2>&1`
+
+# Check if the scp succeeded or failed
+if ! grep -q "No such file or directory" $SCPLOG; then
+ echo "SCP from remote host completed OK"
+else
+ echo "SCP from remote host FAILED"
+ exit 1
+fi
+
+echo "Restoring db from $FULLBACKUPDIR/$FULL/"
+for filename in $FULLBACKUPDIR/$FULL/*; do $SCRIPTDIR/zookeeper-backup-restore.sh -f $filename; done
+
+{%- else %}
+
+FULL=`find $BACKUPDIR -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -nr | head -{{ backup.client.restore_latest }} | tail -1`
+echo "Restoring db from $BACKUPDIR/$FULL/"
+for filename in $BACKUPDIR/$FULL/*; do $SCRIPTDIR/zookeeper-backup-restore.sh -f $filename; done
+
+{%- endif %}
diff --git a/zookeeper/files/backup/zookeeper-backup-client-restore.sh b/zookeeper/files/backup/zookeeper-backup-client-restore.sh
new file mode 100644
index 0000000..4c6b5ad
--- /dev/null
+++ b/zookeeper/files/backup/zookeeper-backup-client-restore.sh
@@ -0,0 +1,90 @@
+{%- from "zookeeper/map.jinja" import backup with context %}
+#!/bin/bash
+# Script to restore zookeeper schema and keyspaces from snapshot one by one
+
+# Configuration
+# -------------
+ ZOOKEEPERDIR="/var/lib/zookeeper/version-2/"
+ DBALREADYRESTORED="{{ backup.backup_dir }}/dbrestored"
+
+# Functions
+# ---------
+ function check_dependencies() {
+ # Function to iterate through a list of required executables to ensure
+ # they are installed and executable by the current user.
+ DEPS="awk cat cut echo find getopt grep hostname "
+ DEPS+="mkdir rm sed tar tr "
+ for bin in $DEPS; do
+ $( which $bin >/dev/null 2>&1 ) || NOTFOUND+="$bin "
+ done
+
+ if [ ! -z "$NOTFOUND" ]; then
+ printf "Error finding required executables: ${NOTFOUND}\n" >&2
+ exit 1
+ fi
+ }
+
+ function usage() {
+ printf "Usage: $0 -h\n"
+ printf " $0 -f <backupfile file>\n"
+ printf " -h,--help Print usage and exit\n"
+ printf " -f,--file <backup tar file> REQUIRED: The backup tar file name\n"
+ exit 0
+ }
+
+
+# Validate Input/Environment
+# --------------------------
+ # Great sample getopt implementation by Cosimo Streppone
+ # https://gist.github.com/cosimo/3760587#file-parse-options-sh
+ SHORT='h:f:'
+ LONG='help,file:'
+ OPTS=$( getopt -o $SHORT --long $LONG -n "$0" -- "$@" )
+
+ if [ $? -gt 0 ]; then
+ # Exit early if argument parsing failed
+ printf "Error parsing command arguments\n" >&2
+ exit 1
+ fi
+
+ eval set -- "$OPTS"
+ while true; do
+ case "$1" in
+ -h|--help) usage;;
+ -f|--file) BACKUPFILE="$2"; shift 2;;
+ --) shift; break;;
+ *) printf "Error processing command arguments\n" >&2; exit 1;;
+ esac
+ done
+
+ # Verify required binaries at this point
+ check_dependencies
+
+ # Only a backup file is required
+ if [ ! -r "$BACKUPFILE" ]; then
+ printf "You must provide the location of a snapshot package\n"
+ exit 1
+ fi
+
+ # Need write access to local directory to create dump file
+ if [ ! -w $( pwd ) ]; then
+ printf "You must have write access to the current directory $( pwd )\n"
+ exit 1
+ fi
+
+
+# LOAD BACKUP FILE
+# ----------------
+ # Extract snapshot package
+ tar -xvzf "$BACKUPFILE" -P
+ RC=$?
+
+ if [ $RC -gt 0 ]; then
+ printf "\nBackup file $BACKUPFILE failed to load.\n"
+ exit 1
+ else
+ printf "\nBackup file $BACKUPFILE was succesfully loaded.\n"
+ touch $DBALREADYRESTORED
+ fi
+
+# Fin.
diff --git a/zookeeper/files/backup/zookeeper-backup-client-runner.sh b/zookeeper/files/backup/zookeeper-backup-client-runner.sh
new file mode 100644
index 0000000..a6e7cc8
--- /dev/null
+++ b/zookeeper/files/backup/zookeeper-backup-client-runner.sh
@@ -0,0 +1,99 @@
+{%- from "zookeeper/map.jinja" import backup with context %}
+#!/bin/bash
+# Script to backup zookeeper schema and create snapshot of keyspaces
+
+# Configuration
+# -------------
+ BACKUPDIR="{{ backup.backup_dir }}/full"
+ TMPDIR="$( pwd )/${PROGNAME}.tmp${RANDOM}"
+ TMPLOG="zookeeper-tmplog.log"
+ ZOOKEEPERDIR="/var/lib/zookeeper/version-2/"
+ KEEP={{ backup.client.full_backups_to_keep }}
+ HOURSFULLBACKUPLIFE={{ backup.client.hours_before_full }} # Lifetime of the latest full backup in seconds
+ LOGDIR="/var/log/backups"
+ RSYNCLOG="/var/log/backups/zookeeper-rsync.log"
+
+
+ if [ $HOURSFULLBACKUPLIFE -gt 24 ]; then
+ FULLBACKUPLIFE=$(( 24 * 60 * 60 ))
+ else
+ FULLBACKUPLIFE=$(( $HOURSFULLBACKUPLIFE * 60 * 60 ))
+ fi
+
+# Create backup directory.
+# ------------------------
+
+ if [ ! -d "$BACKUPDIR" ] && [ ! -e "$BACKUPDIR" ]; then
+ mkdir -p "$BACKUPDIR"
+ fi
+
+ if [ ! -d "$LOGDIR" ] && [ ! -e "$LOGDIR" ]; then
+ mkdir -p "$LOGDIR"
+ fi
+
+ # Create temporary working directory. Yes, deliberately avoiding mktemp
+ if [ ! -d "$TMPDIR" ] && [ ! -e "$TMPDIR" ]; then
+ mkdir -p "$TMPDIR"
+ else
+ printf "Error creating temporary directory $TMPDIR"
+ exit 1
+ fi
+
+
+# Backup and create tar archive
+# ------------------------------
+
+ TIMESTAMP=$( date +"%Y%m%d%H%M%S" )
+
+ echo stat | nc localhost 2181 | grep leader > "$TMPDIR/$TMPLOG"
+ RC=$?
+
+ mkdir -p "$BACKUPDIR/$TIMESTAMP"
+
+ if [ $RC -gt 0 ] && [ ! -s "$TMPDIR/$TMPLOG" ]; then
+ printf "Not a zookeper leader. This script does backup just on zookeper leader.\n"
+ [ "$TMPDIR" != "/" ] && rm -rf "$TMPDIR"
+ exit 0
+ else
+ # Include the timestamp in the filename
+ FILENAME="$BACKUPDIR/$TIMESTAMP/zookeeper-$TIMESTAMP.tar.gz"
+
+ tar -zcvf $FILENAME -P $ZOOKEEPERDIR > /dev/null 2>&1
+ RC=$?
+
+ if [ $RC -gt 0 ]; then
+ printf "Error generating tar archive.\n"
+ [ "$TMPDIR" != "/" ] && rm -rf "$TMPDIR"
+ exit 1
+ else
+ printf "Successfully created a backup tar file.\n"
+ [ "$TMPDIR" != "/" ] && rm -rf "$TMPDIR"
+ fi
+ fi
+
+# rsync just the new or modified backup files
+# -------------------------------------------
+
+ {%- if backup.client.target is defined %}
+ echo "Adding ssh-key of remote host to known_hosts"
+ ssh-keygen -R {{ backup.client.target.host }} 2>&1 | > $RSYNCLOG
+ ssh-keyscan {{ backup.client.target.host }} >> ~/.ssh/known_hosts 2>&1 | >> $RSYNCLOG
+ echo "Rsyncing files to remote host"
+ /usr/bin/rsync -rhtPv --rsync-path=rsync --progress $BACKUPDIR/* -e ssh zookeeper@{{ backup.client.target.host }}:$BACKUPDIR >> $RSYNCLOG
+
+ if [ -s $RSYNCLOG ] && ! grep -q "rsync error: " $RSYNCLOG; then
+ echo "Rsync to remote host completed OK"
+ else
+ echo "Rsync to remote host FAILED"
+ exit 1
+ fi
+
+ {%- endif %}
+
+# Cleanup
+# ---------
+ echo "Cleanup. Keeping only $KEEP full backups"
+ AGE=$(($FULLBACKUPLIFE * $KEEP / 60))
+ find $BACKUPDIR -maxdepth 1 -type d -mmin +$AGE -execdir echo "removing: "$BACKUPDIR/{} \; -execdir rm -rf $BACKUPDIR/{} \;
+
+# Fin.
diff --git a/zookeeper/files/backup/zookeeper-backup-server-restore-call.sh b/zookeeper/files/backup/zookeeper-backup-server-restore-call.sh
new file mode 100644
index 0000000..aa4a7fe
--- /dev/null
+++ b/zookeeper/files/backup/zookeeper-backup-server-restore-call.sh
@@ -0,0 +1,20 @@
+{%- from "zookeeper/map.jinja" import backup with context %}
+#!/bin/sh
+
+# This script is called remotely by zookeeper 'client role' node and returns appropriate backup that client will restore
+
+if [ $# -eq 0 ]; then
+ echo "No arguments provided"
+ exit 1
+fi
+
+# if arg is not an integer
+case $1 in
+ ''|*[!0-9]*) echo "Argument must be integer"; exit 1 ;;
+ *) ;;
+esac
+
+BACKUPDIR="{{ backup.backup_dir }}/full/"
+FULL=`find $BACKUPDIR -mindepth 1 -maxdepth 1 -type d -printf "%P\n" | sort -nr | head -$1 | tail -1`
+
+echo "$BACKUPDIR/$FULL/"
diff --git a/zookeeper/files/backup/zookeeper-backup-server-runner.sh b/zookeeper/files/backup/zookeeper-backup-server-runner.sh
new file mode 100644
index 0000000..eea882c
--- /dev/null
+++ b/zookeeper/files/backup/zookeeper-backup-server-runner.sh
@@ -0,0 +1,21 @@
+{%- from "zookeeper/map.jinja" import backup with context %}
+#!/bin/bash
+
+# Script to erase old backups on zookeeper 'server role' node.
+# ---------
+
+ BACKUPDIR="{{ backup.backup_dir }}/full"
+ KEEP={{ backup.server.full_backups_to_keep }}
+ HOURSFULLBACKUPLIFE={{ backup.server.hours_before_full }} # Lifetime of the latest full backup in seconds
+
+ if [ $HOURSFULLBACKUPLIFE -gt 24 ]; then
+ FULLBACKUPLIFE=$(( 24 * 60 * 60 ))
+ else
+ FULLBACKUPLIFE=$(( $HOURSFULLBACKUPLIFE * 60 * 60 ))
+ fi
+
+# Cleanup
+# ---------
+ echo "Cleanup. Keeping only $KEEP full backups"
+ AGE=$(($FULLBACKUPLIFE * $KEEP / 60))
+ find $BACKUPDIR -maxdepth 1 -type d -mmin +$AGE -execdir echo "removing: "$BACKUPDIR/{} \; -execdir rm -rf $BACKUPDIR/{} \;
diff --git a/zookeeper/init.sls b/zookeeper/init.sls
index 9f86cdc..f9ba06f 100644
--- a/zookeeper/init.sls
+++ b/zookeeper/init.sls
@@ -3,4 +3,7 @@
{%- if pillar.zookeeper.server is defined %}
- zookeeper.server
{%- endif %}
+{%- if pillar.zookeeper.backup is defined %}
+- zookeeper.backup
+{%- endif %}
{%- endif %}
diff --git a/zookeeper/map.jinja b/zookeeper/map.jinja
index f34745c..83b2f1a 100644
--- a/zookeeper/map.jinja
+++ b/zookeeper/map.jinja
@@ -18,6 +18,21 @@
base: /usr/lib/zookeeper
services:
- zookeeper
+
+backup:
+ Debian:
+ pkgs:
+ - rsync
+ backup_dir: '/var/backups/zookeeper'
+ cron: True
+ RedHat:
+ pkgs:
+ - rsync
+ backup_dir: '/var/backups/zookeeper'
+ cron: True
+
{%- endload %}
-{%- set server = salt['grains.filter_by'](base_defaults, merge=salt['pillar.get']('zookeeper:server')) %}
\ No newline at end of file
+{%- set server = salt['grains.filter_by'](base_defaults, merge=salt['pillar.get']('zookeeper:server')) %}
+
+{% set backup = salt['grains.filter_by'](base_defaults['backup'], merge=salt['pillar.get']('zookeeper:backup', {}), base='backup') %}