From 915fb4e6fe6695bc6c4011c3fd43577a55c8d6f4 Mon Sep 17 00:00:00 2001 From: Cerem Cem ASLAN Date: Sat, 28 Jan 2023 12:51:15 +0300 Subject: [PATCH] added Responsive Remote Development support --- README.md | 39 ++++++---- sync-config-example.sh | 5 ++ sync-with-sgw.sh | 158 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 12 deletions(-) create mode 100644 sync-config-example.sh create mode 100755 sync-with-sgw.sh diff --git a/README.md b/README.md index 377b77d..7b62237 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,15 @@ # Description -This toolset is intended to use for managing remote Linux devices (RaspberryPi in mind, but any remote Linux system will work) from host Linux systems, by basically simplifying 6 tasks, where you need to: +This toolset is intended for administrating remote Linux devices that are directly connected or behind a proxy server (RaspberryPi in mind, but any remote Linux system will work), by simplifying 7 tasks: -1. ..make `ssh` for performing remote tasks (either directly or over a link up server). -2. ..use simple drag and drop style file transfers (by `sshfs`). -3. ..backup the target's entire root filesystem (by `rsync`). -4. ..create fast and efficient **differential full backups** (by hardlinks or by BTRFS snapshots). -5. ..create a separate bootable system disk from any of your backups. -6. ..clone a target with a new identity. +1. `make ssh` to connect the remote shell (either directly or over a link up server). +2. Responsively edit remote files via local IDE almost independent from the internet connection speed and interruptions ("Responsive remote development"). +3. Use simple drag and drop style file transfers (by `sshfs`). +4. Backup the target's entire root filesystem (by `rsync`). +5. Create fast and efficient **differential full backups** (by hardlinks or by BTRFS snapshots). +6. Create a separate physical bootable system disk from any of your backups. +7. Clone the current device with a new identity to create a new device. -This simplification is achieved by: - - * Placing separate scripts for each task described above and providing a simple `Makefile`. - * Keeping the scripts, configuration files and backups in a folder called `your-project`. # Install @@ -67,6 +64,22 @@ make ssh Makes ssh connection either directly or via the link up server according to [your connection type](#set-connection-type). +### Responsive Remote Development + +Responsive remote development means keeping a local folder in sync with a remote folder. + +1. `cp ./sync-config-example.sh path/to/your/project/folder/my-sync-config.sh` +2. Edit `my-sync-config.sh` accordingly. See `./sync-with-sgw.sh --help` for options. +3. Send your project folder to your remote system and watch for changes by: + + ./sync-with-sgw.sh -c path/to/your/project/folder/my-sync-config.sh --dry-run + +This will keep `path/to/your/project/folder/` and `$dest_dir` (within your config file) in sync. Remove the `--dry-run` switch for real transfer if the transfer summary is as you expected. + +Synchronization will exclude the `.git` folder and the other files/folders listed in `path/to/your/project/folder/.gitignore`. + +`run_before_sync` hooks can be used to build, bundle, copy files or perfom any other tasks before the actual synchronization. Synchronization will fail and display a visual error message if any of the hooks fails. + ### Mount target root ```bash @@ -74,7 +87,9 @@ make mount-root ``` Mounts the root folder to `your-project/NODE_ROOT`, which you can use for drag-n-drop style file transfers. -You can later unmount with `make umount-root` without using `sudo` command. +You can later unmount with `make umount-root` without using `sudo` command. + +This feature is only practical with fast (generally on local) connections. ### Sync target's root folder diff --git a/sync-config-example.sh b/sync-config-example.sh new file mode 100644 index 0000000..4c27fa8 --- /dev/null +++ b/sync-config-example.sh @@ -0,0 +1,5 @@ +proxy_host=aktos1 +dest_host=aea@localhost:7104 +dest_dir='~/apps/xy2-100-v3/' +run_before_sync+=("make all") +use_gitignore=true \ No newline at end of file diff --git a/sync-with-sgw.sh b/sync-with-sgw.sh new file mode 100755 index 0000000..64883a4 --- /dev/null +++ b/sync-with-sgw.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +_sdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +set -eu + +show_help(){ + cat <&2 echo + >&2 echo "$@" + exit 1 +} + +help_die(){ + >&2 echo + >&2 echo "$@" + show_help + exit 1 +} + +# Parse command line arguments +# --------------------------- +# Initialize parameters +config= +dry_run= +# --------------------------- +args_backup=("$@") +args=() +_count=1 +while [ $# -gt 0 ]; do + key="${1:-}" + case $key in + -h|-\?|--help|'') + show_help # Display a usage synopsis. + exit + ;; + # -------------------------------------------------------- + -c|--config) shift + config="$1" + ;; + --dry-run) + dry_run="--dry-run" + ;; + # -------------------------------------------------------- + -*) # Handle unrecognized options + help_die "Unknown option: $1" + ;; + *) # Generate the new positional arguments: $arg1, $arg2, ... and ${args[@]} + if [[ ! -z ${1:-} ]]; then + declare arg$((_count++))="$1" + args+=("$1") + fi + ;; + esac + [[ -z ${1:-} ]] && break || shift +done; set -- "${args_backup[@]-}" +# Use $arg1 in place of $1, $arg2 in place of $2 and so on, +# "$@" is in the original state, +# use ${args[@]} for new positional arguments + +[[ -f "$config" ]] || die "Configuration file is required." +source "$config" + +SRC_DIR="$(dirname "$config")" + +read SGW_USERNAME SGW_HOST SGW_PORT_ON_SERVER <<< $(echo $dest_host | sed 's/@/ /' | sed 's/:/ /') +[[ -z $SGW_HOST ]] && { SGW_HOST=$SGW_USERNAME; SGW_USERNAME=''; } +read PROXY_USERNAME PROXY_HOST PROXY_PORT <<< $(echo $proxy_host | sed 's/@/ /' | sed 's/:/ /') +[[ -z $PROXY_HOST ]] && { PROXY_HOST=$PROXY_USERNAME; PROXY_USERNAME=''; } + +ignores=(--exclude '.git') +gitignore_file="$SRC_DIR/.gitignore" +if ${use_gitignore:-false} && [[ -f "$gitignore_file" ]]; then + while IFS=: read -r line; do + ignores+=(--exclude "$line") + done < <(grep "" "$gitignore_file") +fi + +script_name="$(basename $0)" + +echo_blue () { + echo -e "\e[1;34m$*\e[0m" +} + +echo_yellow () { + echo -e "\e[1;33m$*\e[0m" +} + +echo_green () { + echo -e "\e[1;32m$*\e[0m" +} + +RSYNC="nice -n19 ionice -c3 rsync" + +timestamp(){ + date +'%Y-%m-%d %H:%M' +} + +previous_sync_failed=false +while :; do + hook_failed=false + for cmd in "${run_before_sync[@]}"; do + echo_blue "Running hook before sync: $cmd" + eval $cmd || { hook_failed=true; break; } + done + + if ! $hook_failed; then + echo_blue "$(timestamp): Synchronizing..." + + [[ -z $dry_run ]] || set -x + if $RSYNC -avzhP --delete $dry_run "${ignores[@]}" \ + -e "ssh -A -J ${PROXY_HOST} -p ${SGW_PORT_ON_SERVER}" \ + "$SRC_DIR" \ + ${SGW_USERNAME}@localhost:"${dest_dir}"; then + + $previous_sync_failed && notify-send -u critical "$script_name Succeeded." "$(timestamp)" + else + period=10 + $previous_sync_failed || notify-send -u critical "$script_name Failed." "Retrying in $period seconds." + sleep $period + echo_yellow "Retrying..." + previous_sync_failed=true + continue + fi + $previous_sync_failed || notify-send "Sync done." "$(timestamp): ${dest_dir}" + + previous_sync_failed=false + else + notify-send -u critical "$script_name Failed." "$cmd failed ($(timestamp))" + fi + + echo_green "Waiting for directory changes..." + inotifywait -q -e modify,create,delete -r "$SRC_DIR" +done