functions/rsync.sh

#!/bin/bash -xe

export LANG=C

# define this vars before use
FILESDIR=${FILESDIR:-"snapshots"}
LATESTSUFFIX=${LATESTSUFFIX:-"-latest"}

export TIMESTAMP=${TIMESTAMP:-$(date "+%Y-%m-%d-%H%M%S")}
export SAVE_LAST_DAYS=${SAVE_LAST_DAYS:-61}
export WARN_DATE=$(date "+%Y%m%d" -d "$SAVE_LAST_DAYS days ago")

get_empty_dir() {
    mktemp -d
}

get_symlink() {
    local LINKDEST=$1
    local LINKNAME=$(mktemp -u)
    ln -s --force "$LINKDEST" "$LINKNAME" && echo "$LINKNAME"
}

rsync_list() {
    local RSYNCPATH=$1
    local TEMPFILE=$(mktemp)
    local RESULT=1
    if rsync -l "${RSYNCPATH}/" 2>/dev/null > "$TEMPFILE" ; then
        grep -v '\.$' "$TEMPFILE" || :
        RESULT=0
    fi
    rm "$TEMPFILE"
    return "$RESULT"
}

rsync_list_links() {
    local RSYNCPATH=$1
    local TEMPFILE=$(mktemp)
    local RESULT=1
    if rsync_list "$RSYNCPATH" > "$TEMPFILE" ; then
        grep '^l' "$TEMPFILE" | awk '{print $(NF-2)" "$NF}' || :
        RESULT=0
    fi
    rm "$TEMPFILE"
    return "$RESULT"
}

rsync_list_dirs() {
    local RSYNCPATH=$1
    local TEMPFILE=$(mktemp)
    local RESULT=1
    if rsync_list "$RSYNCPATH" > "$TEMPFILE" ; then
        grep '^d' "$TEMPFILE" | awk '{print $NF}' || :
        RESULT=0
    fi
    rm "$TEMPFILE"
    return "$RESULT"
}

rsync_list_files() {
    local RSYNCPATH=$1
    local TEMPFILE=$(mktemp)
    local RESULT=1
    if rsync_list "$RSYNCPATH" > "$TEMPFILE" ; then
        grep -vE '^d|^l' "$TEMPFILE" | awk '{print $NF}' || :
        RESULT=0
    fi
    rm "$TEMPFILE"
    return "$RESULT"
}

rsync_delete_file() {
    local RSYNCPATH=${1%/}
    local FILENAME=${RSYNCPATH##*/}
    local EMPTYDIR=$(get_empty_dir)
    if rsync_list_files "$RSYNCPATH" &>/dev/null ; then
        [ -n "$IS_VERBOSE" ] && echo "[info] Deleting file ${RSYNCPATH}"
        rsync -rv --delete --include="$FILENAME" '--exclude=*' \
            "${EMPTYDIR}/" "${RSYNCPATH%/*}/"
        [ -d "$EMPTYDIR" ] && rm -rf "$EMPTYDIR"
    fi
}

rsync_delete_dir() {
    local RSYNCPATH=$1
    if rsync_list "${RSYNCPATH}" &>/dev/null ; then
        local EMPTYDIR=$(get_empty_dir)
        [ -n "$IS_VERBOSE" ] && echo "[info] Deleting directory ${RSYNCPATH}/"
        rsync --delete -a "${EMPTYDIR}/" "${RSYNCPATH}/" \
            && rsync_delete_file "$RSYNCPATH"
        [ -d "$EMPTYDIR" ] && rm -rf "$EMPTYDIR"
    fi
}

rsync_create_dir() {
    local RSYNCPATH=$1
    if ! rsync_list_dirs "${RSYNCPATH%/*}" &>/dev/null ; then
        [ "${RSYNCPATH%/*}" == "${RSYNCPATH}" ] && fail_exit "[FAIL] Can't reach to the rsync root"
        rsync_create_dir "${RSYNCPATH%/*}"
    fi
    if ! rsync_list_dirs "${RSYNCPATH}" &>/dev/null ; then
        [ -n "$IS_VERBOSE" ] && echo "[info] Creating directory ${RSYNCPATH}/"
        local EMPTYDIR=$(get_empty_dir)
        rsync -a "${EMPTYDIR}/" "${RSYNCPATH}/"
        [ -d "$EMPTYDIR" ] && rm -rf "$EMPTYDIR"
    fi
}

rsync_create_symlink() {
    # Create symlink $1 -> $2
    # E.g. "create_symlink host repos/6.1 files/6.1-stable"
    # wll create symlink repos/6.1 -> repos/files/6.1-stable
    local RSYNCPATH=$1
    local LINKDEST=$2
    [ -n "$IS_VERBOSE" ] && echo "[info] Creating symlink $RSYNCPATH to $LINKDEST"
    local SYMLINK_FILE=$(get_symlink "$LINKDEST")
    rsync -vl "$SYMLINK_FILE" "${RSYNCPATH}"
    rm "$SYMLINK_FILE"

    # Make text file for dereference symlinks
    local TARGET_TXT_FILE=$(mktemp)
    echo "$LINKDEST" > "$TARGET_TXT_FILE"
    rsync -vl "$TARGET_TXT_FILE" "${RSYNCPATH}.target.txt"
    rm "$TARGET_TXT_FILE"
}

#######################################################
rsync_remove_old_snapshots() {
    # Remove mirrors older then $SAVE_LAST_DAYS and w/o symlinks on it
    local RSYNCPATH=${1%%+(/)}

    local FOLDERNAME=${RSYNCPATH##*/}
    local DIRS=$(rsync_list_dirs "${RSYNCPATH%/*}/$FILESDIR" | grep "^$FOLDERNAME\-" )
    local dir
    for dir in $DIRS; do
        local ddate=$(echo "$dir" | awk -F '[-]' '{print $(NF-3)$(NF-2)$(NF-1)}')
        [ "$ddate" -gt "$WARN_DATE" ] && continue
        LINKS=$(rsync_list_links "${RSYNCPATH%/*}/$FILESDIR" | grep " $dir$" \
            | awk '{print $1}'; \
            rsync_list_links "${RSYNCPATH%/*}" | grep " ${FILESDIR}/$dir$" \
            | awk '{print $1}')
        if [ "$LINKS" = "" ]; then
            [ -n "$IS_VERBOSE" ] \
                && echo "[info] Delete snapshot $dir as obsoleted"
            rsync_delete_dir "${RSYNCPATH%/*}/${FILESDIR}/$dir"
            continue
        fi
        [ -n "$IS_VERBOSE" ] \
            && echo "[info] $dir Can't be deleted because of synlinks: $LINKS"
    done
}

######################################################
rsync_transfer() {
    # sync files to remote host
    # $1 - remote path
    # $2 - source dir
    # $3 - dest dir name (the same as source dir name if empty)
    # $4 - extra params for rsync (optional)
    local SRCDIR=${1%%+(/)}
    local RSYNCPATH=${2%%+(/)}
    local FOLDERNAME=${3%%+(/)}
    local RSYNC_EXTRA_PARAMS=$4
    [ -z "$FOLDERNAME" ] && FOLDERNAME=${SRCDIR##*/}

    # Lock source dir to prevent concurent sync stage
    local LOCK_FILE=${SRCDIR}.lock
    job_lock "$LOCK_FILE" wait
    rsync_list_dirs "${RSYNCPATH}/$FILESDIR" &>/dev/null \
        || rsync_create_dir "${RSYNCPATH}/$FILESDIR"

    # Skip host part of RSYNCPATH
    local RSYNCROOT
    if [ "${RSYNCPATH/:\/\/}" != "$RSYNCPATH" ] ; then 
        # `rsync://[user@]host[:port]/module/*` case
        RSYNCROOT=$(echo "$RSYNCPATH" | awk -F'/' \
            '{ for(i=5;i<=NF;++i) printf("/" $i) }')
    else
        local _first_part=${RSYNCPATH%%/*}
        if [ "${_first_part/::}" != "$_first_part" ] ; then
            # `[user@]host::module/*` case
            RSYNCROOT=$(echo "$RSYNCPATH" | awk -F'::' '{ print $2 }')
            RSYNCROOT="/${RSYNCROOT#*/}"
        elif [ "${_first_part/:}" != "$_first_part" ] ; then
            # `[user@]host:*` case
            RSYNCROOT=$(echo "$RSYNCPATH" | awk -F':' '{ print $2 }')
        else
            # local path case
            RSYNCROOT=$RSYNCPATH
        fi
    fi

    # Remove leading slashes
    RSYNCROOT=${RSYNCROOT##+(/)}

    local OPTIONS="--archive --verbose --force --ignore-errors --delete-excluded --no-owner --no-group \
          $RSYNC_EXTRA_PARAMS --delete --link-dest=/${RSYNCROOT}/${FILESDIR}/${FOLDERNAME}$LATESTSUFFIX"

    local SNAPSHOT_NAME=${FOLDERNAME}-$TIMESTAMP
    local DEST_DIR=${RSYNCPATH}/${FILESDIR}/$SNAPSHOT_NAME
    local LATEST_LINK=${FILESDIR}/${FOLDERNAME}$LATESTSUFFIX

    local RESULT=1
    # shellcheck disable=SC2086
    if rsync $OPTIONS "${SRCDIR}/" "$DEST_DIR" ; then
        rsync_delete_file "${RSYNCPATH}/$LATEST_LINK" \
        && rsync_create_symlink "${RSYNCPATH}/$LATEST_LINK" "$SNAPSHOT_NAME" \
        && rsync_delete_file "${RSYNCPATH}/$FOLDERNAME" \
        && rsync_create_symlink "${RSYNCPATH}/$FOLDERNAME" "${FILESDIR}/$SNAPSHOT_NAME" \
        && rsync_remove_old_snapshots "${RSYNCPATH}/$FOLDERNAME"
        RESULT=0
    else
        rsync_delete_dir "${RSYNCPATH}/$DEST_DIR"
    fi

    # Unlock source dir
    job_lock "$LOCK_FILE" unset

    return $RESULT
}