sync-rclone

#!/bin/bash
#
# This script synchronizes storage buckets, from local to remote or vice versa,
# making use of rclone, https://github.com/ncw/rclone.
#
# By default this script syncs from local directory to the remote Google Cloud.
#
#  $ sync-rclone
#
# Usually you need to set $SRC_REPO to an absolute path to a local directory,
# and $DST_REPO to a remote Google Storage repository.
#
#  $ SRC_REPO=/tmp/some-dir DST_REPO=gcs:otherproj sync-rclone
#
# Alternatively, you can also set them the other way around, i.e. $SRC_REPO to
# a remote one, $DST_REPO to a local one.
#
#  $ SRC_REPO=gcs:otherproj DST_REPO=/tmp/some-dir sync-rclone
#
# Note that the remote repo name needs to be a format of REPO_NAME:BUCKET_NAME.
# such as "gcs:flatcar-jenkins".
#
# Set custom GCS configs, e.g.:
#
#  $ GCS_PROJECT_NUMBER=987654321098 sync-rclone
#  $ GCS_SERVICE_ACCOUNT_CONFIG=/tmp/sa.json sync-rclone
#
# You can set DRY_RUN to 1, to only show commands without running them actually.
#
#  $ DRY_RUN=1 SRC_REPO=/tmp/some-dir DST_REPO=gcs:otherproj sync-rclone
#
# You can set FORCE_MODE to 1, to be able to overwrite data on the dest repo.
#
#  $ FORCE_MODE=1 SRC_REPO=/tmp/some-dir DST_REPO=gcs:otherproj sync-rclone

set -euo pipefail

readonly RCLONE_CONFIGDIR="$HOME/.config/rclone"
readonly SRC_REPO="${SRC_REPO:-}"
readonly DST_REPO="${DST_REPO:-}"
readonly DRY_RUN="${DRY_RUN:-0}"
readonly FORCE_MODE="${FORCE_MODE:-0}"

if [[ -z "${SRC_REPO}" ]]; then
    echo "ERROR: please set a valid SRC_REPO variable, such as /var/www/origin.release.flatcar-linux.net/alpha."
    exit 1
fi

if [[ -z "${DST_REPO}" ]]; then
    echo "ERROR: please set a valid DST_REPO variable, such as gcs:flatcar-jenkins."
    exit 1
fi

readonly GCS_PROJECT_NUMBER="${GCS_PROJECT_NUMBER:-5257126083}"  # NOTE: please fill in a valid project number
readonly GCS_SERVICE_ACCOUNT_CONFIG="${GCS_SERVICE_ACCOUNT_CONFIG:-${RCLONE_CONFIGDIR}/flatcar.json}"

function pushd {
    command pushd "$@" > /dev/null
}

function popd {
    command popd > /dev/null
}

function install_rclone {
    local RCLONE_DESTDIR="/opt/bin"

    pushd /tmp
    curl -L https://downloads.rclone.org/rclone-current-linux-amd64.zip -o /tmp/rclone-current-linux-amd64.zip
    unzip rclone-current-linux-amd64.zip

    pushd rclone-*-linux-amd64
    mkdir --parents ${RCLONE_DESTDIR}
    sudo cp rclone ${RCLONE_DESTDIR}/rclone
    sudo chown root:root ${RCLONE_DESTDIR}/rclone
    sudo chmod 755 ${RCLONE_DESTDIR}/rclone
    popd

    rm -rf /tmp/rclone-*-linux-amd64*
    popd
}

function gen_rclone_config {
    # Please fill in below a valid project_number to be able to configure
    # Google Cloud storage.
    # Also service_account_file needs to be a path to a valid json file
    # downloaded from the console of Google Cloud.
    # See also https://rclone.org/googlecloudstorage.

    mkdir --parents "${RCLONE_CONFIGDIR}"

    tee "${RCLONE_CONFIGDIR}/rclone.conf" << EOF
[gcs]
type = google cloud storage
client_id = 
client_secret = 
project_number = ${GCS_PROJECT_NUMBER}
service_account_file = ${GCS_SERVICE_ACCOUNT_CONFIG}
object_acl = bucketOwnerFullControl
bucket_acl = authenticatedRead
location = eu
storage_class = 
token = 
EOF
}

# normalize_path returns a valid representation of one of the following paths:
#  * In case of a normal local path, return its absolute path
#  * In case of an rclone repo representation, return the original repo string.
function normalize_path() {
    local repopath=$1

    if ! [[ "${repopath}" =~ ":" ]]; then
        repopath=$(realpath -m "${repopath}")
    fi

    echo "${repopath}"
}

# repo_exists checks if a repo exists.
#  * In case of a normal local path, return 1 if a path exists locally.
#  * In case of an rclone repo representation, return 1 if a remote path
#    exists on the remote repo.
function repo_exists() {
    local ret=0
    local repopath=$1

    if [[ "$repopath" =~ ":" ]]; then
        [[ $(rclone ls "${repopath}") ]] && ret=1
    else
        repopath=$(realpath -m "${repopath}")
        [[ -e "${repopath}" ]] && ret=1
    fi

    echo "${ret}"
}

readonly TIMESTAMP="$(date +%Y%m%d%H%M%S)"
readonly DST_REPO_NORM=$(normalize_path "${DST_REPO}")
RCLONE_OPTIONS="--verbose --backup-dir=${DST_REPO_NORM}-backup-${TIMESTAMP}"

if [[ "${DRY_RUN}" -ne 0 ]]; then
    RCLONE_OPTIONS+=" --dry-run"
fi

if ! which rclone >/dev/null 2>&1; then
    install_rclone
fi

if [[ ! -f "${RCLONE_CONFIGDIR}/rclone.conf" ]]; then
    gen_rclone_config
fi

if [[ "${FORCE_MODE}" -eq 0 ]]; then
    if [[ $(repo_exists "${DST_REPO}") -eq 1 ]]; then
        echo "${DST_REPO} already exists."
        echo "If you really know what you are doing, please set FORCE_MODE=1."
        exit 1
    fi
fi

# shellcheck disable=SC2086
exec rclone sync ${RCLONE_OPTIONS} "${SRC_REPO}" "${DST_REPO}"