Compare commits
4
Commits
899eb11a2d
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3c0639f940 | ||
|
|
ea149aa082 | ||
|
|
fac0d3202b | ||
|
|
fc26387aeb |
No files matched your search
+179
-54
@@ -3,34 +3,57 @@
|
||||
# Script Name: AppBackup-restic.sh
|
||||
# Description:
|
||||
# Creates a backup of a web application and optionally its MySQL database.
|
||||
# The script archives the application home directory and uploads the backup
|
||||
# either to a remote SFTP server or to a Backblaze B2 repository using Restic,
|
||||
# depending on the host on which it is executed.
|
||||
# The script archives the application home directory locally and backs it
|
||||
# up to a Backblaze B2 repository using Restic.
|
||||
#
|
||||
# Usage:
|
||||
# AppBackup-restic.sh --appname <name> --appuser <user> [--apppassword <password>]
|
||||
#
|
||||
# Author: Charlie Silverman
|
||||
# Version: 1.7
|
||||
# Version: 20261031-01
|
||||
#
|
||||
# Requirements:
|
||||
# restic, jq, awk, tar, mysqldump (when --apppassword is used),
|
||||
# mailx or mail (for email notifications), numfmt (optional, for
|
||||
# human-readable sizes).
|
||||
# Restic environment file: /root/.restic-env
|
||||
#
|
||||
# Versioning:
|
||||
# Versions use the format YYYYMMDD-xx, where YYYYMMDD is the release date
|
||||
# and xx is the release number for that day, starting at 01.
|
||||
#
|
||||
# Revision History:
|
||||
# 1.0 - Initial version.
|
||||
# 1.1 - Added logging to file.
|
||||
# 1.2 - Added strict error handling, input validation, and FQDN support.
|
||||
# 1.3 - Added trap-based error logging and improved operational visibility.
|
||||
# 1.4 - Added application-specific log files, interrupt handling,
|
||||
# and enhanced logging.
|
||||
# 1.5 - Added command-line help.
|
||||
# 1.6 - Replaced positional parameters with named command-line options
|
||||
# and added support for short/long option formats.
|
||||
# 1.7 - Added email delivery of current execution log on success,
|
||||
# failure, or interruption.
|
||||
# 1.0 - Initial version.
|
||||
# 1.1 - Added logging to file.
|
||||
# 1.2 - Added strict error handling, input validation, and FQDN support.
|
||||
# 1.3 - Added trap-based error logging and improved operational visibility.
|
||||
# 1.4 - Added application-specific log files, interrupt handling,
|
||||
# and enhanced logging.
|
||||
# 1.5 - Added command-line help.
|
||||
# 1.6 - Replaced positional parameters with named command-line options
|
||||
# and added support for short/long option formats.
|
||||
# 1.7 (a) - Added --no-tablespaces to mysqldump. Added --single-transaction
|
||||
# to eliminate the need to lock tables during backup.
|
||||
# 1.7 (b) - Added email delivery of current execution log on success,
|
||||
# failure, or interruption.
|
||||
# 20261031-01 - Consolidated the two divergent 1.7 versions into a single
|
||||
# script. Adopted YYYYMMDD-xx versioning scheme. Version is now
|
||||
# defined once (SCRIPT_VERSION) and used in help and logs.
|
||||
# Removed SFTP upload; backups always go to Restic.
|
||||
# Restic snapshot details (ID, parent, tags, file/dir counts,
|
||||
# data added, snapshot size, duration) are now written to the
|
||||
# logs. restic backup now runs with --json and the details are
|
||||
# read from its backup summary. Added required-command check.
|
||||
# Temporary restic output is removed on failure or interrupt.
|
||||
#
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_VERSION="20261031-01"
|
||||
NOTIFY_EMAIL="ackitsme3@gmail.com"
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Help Function
|
||||
#------------------------------------------------------------------------------
|
||||
@@ -39,12 +62,12 @@ show_help() {
|
||||
|
||||
cat << EOF
|
||||
|
||||
AppBackup-restic.sh v1.7
|
||||
AppBackup-restic.sh v${SCRIPT_VERSION}
|
||||
|
||||
DESCRIPTION
|
||||
Creates a backup of a web application's files and optionally its
|
||||
MySQL database. The backup is uploaded either via SFTP or Restic,
|
||||
depending on the host.
|
||||
MySQL database. The backup is sent to a Backblaze B2 repository
|
||||
using Restic.
|
||||
|
||||
USAGE
|
||||
|
||||
@@ -62,7 +85,7 @@ OPTIONS
|
||||
Required.
|
||||
|
||||
-p, --apppassword <password>
|
||||
MySQL password and SFTP password (if applicable).
|
||||
MySQL password. If omitted, the database backup is skipped.
|
||||
Optional.
|
||||
|
||||
-h, --help
|
||||
@@ -70,13 +93,13 @@ OPTIONS
|
||||
|
||||
EXAMPLES
|
||||
|
||||
AppBackup-restic.sh \
|
||||
--appname KISNetManager \
|
||||
--appuser kisuser \
|
||||
AppBackup-restic.sh \\
|
||||
--appname KISNetManager \\
|
||||
--appuser kisuser \\
|
||||
--apppassword MySecurePassword
|
||||
|
||||
AppBackup-restic.sh \
|
||||
-a CustomerPortal \
|
||||
AppBackup-restic.sh \\
|
||||
-a CustomerPortal \\
|
||||
-u portaluser
|
||||
|
||||
LOG FILES
|
||||
@@ -91,7 +114,7 @@ EMAIL NOTIFICATIONS
|
||||
|
||||
Current execution logs are emailed to:
|
||||
|
||||
ackitsme3@gmail.com
|
||||
${NOTIFY_EMAIL}
|
||||
|
||||
Success, failure, and interruption events each generate
|
||||
a separate email.
|
||||
@@ -102,7 +125,7 @@ AUTHOR
|
||||
|
||||
VERSION
|
||||
|
||||
1.7
|
||||
${SCRIPT_VERSION}
|
||||
|
||||
EOF
|
||||
|
||||
@@ -188,6 +211,7 @@ SAFE_APPNAME=$(echo "$AppName" | tr -cs '[:alnum:]_-' '_')
|
||||
|
||||
LOGFILE="/tmp/AppBackup-${SAFE_APPNAME}.log"
|
||||
RUNLOG="/tmp/AppBackup-${SAFE_APPNAME}-${DATE}.log"
|
||||
RESTIC_OUTPUT=""
|
||||
|
||||
touch "$RUNLOG"
|
||||
|
||||
@@ -214,11 +238,11 @@ send_email_log() {
|
||||
|
||||
if command -v mailx >/dev/null 2>&1; then
|
||||
|
||||
mailx -s "$subject" ackitsme3@gmail.com < "$RUNLOG"
|
||||
mailx -s "$subject" "$NOTIFY_EMAIL" < "$RUNLOG"
|
||||
|
||||
elif command -v mail >/dev/null 2>&1; then
|
||||
|
||||
mail -s "$subject" ackitsme3@gmail.com < "$RUNLOG"
|
||||
mail -s "$subject" "$NOTIFY_EMAIL" < "$RUNLOG"
|
||||
|
||||
else
|
||||
|
||||
@@ -227,6 +251,80 @@ send_email_log() {
|
||||
fi
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Convert a byte count to a human-readable size (e.g. 1.23GiB)
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
human_bytes() {
|
||||
|
||||
local bytes="${1:-0}"
|
||||
|
||||
if command -v numfmt >/dev/null 2>&1; then
|
||||
numfmt --to=iec-i --suffix=B --format="%.2f" "$bytes" 2>/dev/null \
|
||||
|| echo "${bytes} B"
|
||||
else
|
||||
echo "${bytes} B"
|
||||
fi
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Write Restic snapshot details from a backup summary (JSON) to the log
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
log_snapshot_info() {
|
||||
|
||||
local summary="$1"
|
||||
|
||||
local snapshot_id parent_id
|
||||
local files_new files_changed files_unmodified
|
||||
local dirs_new dirs_changed dirs_unmodified
|
||||
local data_added data_added_packed
|
||||
local total_files total_bytes total_duration
|
||||
|
||||
snapshot_id=$(jq -r '.snapshot_id // "none"' <<< "$summary")
|
||||
parent_id=$(restic snapshots "$snapshot_id" --json 2>/dev/null \
|
||||
| jq -r '.[0].parent // "none"' 2>/dev/null || echo "unknown")
|
||||
|
||||
files_new=$(jq -r '.files_new // 0' <<< "$summary")
|
||||
files_changed=$(jq -r '.files_changed // 0' <<< "$summary")
|
||||
files_unmodified=$(jq -r '.files_unmodified // 0' <<< "$summary")
|
||||
dirs_new=$(jq -r '.dirs_new // 0' <<< "$summary")
|
||||
dirs_changed=$(jq -r '.dirs_changed // 0' <<< "$summary")
|
||||
dirs_unmodified=$(jq -r '.dirs_unmodified // 0' <<< "$summary")
|
||||
data_added=$(jq -r '.data_added // 0' <<< "$summary")
|
||||
data_added_packed=$(jq -r '.data_added_packed // empty' <<< "$summary")
|
||||
total_files=$(jq -r '.total_files_processed // 0' <<< "$summary")
|
||||
total_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
||||
total_duration=$(jq -r '.total_duration // 0' <<< "$summary")
|
||||
|
||||
log "---------------- Restic Snapshot -----------------"
|
||||
log "Snapshot ID: ${snapshot_id:0:8} (${snapshot_id})"
|
||||
log "Parent snapshot: ${parent_id:0:8}"
|
||||
log "Tags: apache, website, ${AppName}, ${HOSTNAME_FQDN}"
|
||||
log "Files: ${files_new} new, ${files_changed} changed, ${files_unmodified} unmodified"
|
||||
log "Directories: ${dirs_new} new, ${dirs_changed} changed, ${dirs_unmodified} unmodified"
|
||||
if [[ -n "$data_added_packed" ]]; then
|
||||
log "Data added: $(human_bytes "$data_added") ($(human_bytes "$data_added_packed") stored)"
|
||||
else
|
||||
log "Data added: $(human_bytes "$data_added")"
|
||||
fi
|
||||
log "Snapshot size: $(human_bytes "$total_bytes") in ${total_files} files"
|
||||
log "Duration: $(awk -v d="$total_duration" 'BEGIN{printf "%.1f", d}') seconds"
|
||||
log "--------------------------------------------------"
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Remove temporary Restic output
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
cleanup_restic_output() {
|
||||
|
||||
if [[ -n "${RESTIC_OUTPUT:-}" ]]; then
|
||||
rm -f "${RESTIC_OUTPUT}"
|
||||
RESTIC_OUTPUT=""
|
||||
fi
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Error Handling
|
||||
#------------------------------------------------------------------------------
|
||||
@@ -238,6 +336,8 @@ error_handler() {
|
||||
|
||||
log "ERROR: Backup failed on line ${line_number} with exit code ${exit_code}"
|
||||
|
||||
cleanup_restic_output
|
||||
|
||||
send_email_log \
|
||||
"FAILED: AppBackup ${AppName} (${HOSTNAME_FQDN})"
|
||||
|
||||
@@ -248,6 +348,8 @@ interrupt_handler() {
|
||||
|
||||
log "ERROR: Backup interrupted by signal"
|
||||
|
||||
cleanup_restic_output
|
||||
|
||||
send_email_log \
|
||||
"INTERRUPTED: AppBackup ${AppName} (${HOSTNAME_FQDN})"
|
||||
|
||||
@@ -262,7 +364,7 @@ trap interrupt_handler SIGINT SIGTERM
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
log "================================================================="
|
||||
log "Starting backup job"
|
||||
log "Starting backup job (AppBackup-restic.sh v${SCRIPT_VERSION})"
|
||||
log "Application: $AppName"
|
||||
log "Application User: $AppUser"
|
||||
log "Host: $HOSTNAME_FQDN"
|
||||
@@ -270,6 +372,25 @@ log "Historical Log: $LOGFILE"
|
||||
log "Run Log: $RUNLOG"
|
||||
log "================================================================="
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Verify Requirements
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
for cmd in restic jq awk tar; do
|
||||
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
|
||||
log "ERROR: Missing required command: $cmd"
|
||||
|
||||
send_email_log \
|
||||
"FAILED: AppBackup ${AppName} (${HOSTNAME_FQDN})"
|
||||
|
||||
exit 1
|
||||
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Restic Repository Configuration
|
||||
#------------------------------------------------------------------------------
|
||||
@@ -312,6 +433,8 @@ if [[ -n "$AppPassword" ]]; then
|
||||
mkdir -p "/home/${AppUser}/SQLBackup"
|
||||
|
||||
mysqldump \
|
||||
--no-tablespaces \
|
||||
--single-transaction \
|
||||
-u "$AppUser" \
|
||||
--password="$AppPassword" \
|
||||
"$AppName" \
|
||||
@@ -350,37 +473,39 @@ log "Archive created successfully"
|
||||
# Upload Backup
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
if [[ "$HOSTNAME_FQDN" != "ackitserver.ackcentuate.com" ]]; then
|
||||
log "Loading Restic environment"
|
||||
|
||||
if [[ -z "$AppPassword" ]]; then
|
||||
log "ERROR: SFTP upload requires --apppassword"
|
||||
exit 1
|
||||
fi
|
||||
source /root/.restic-env
|
||||
|
||||
log "Uploading archive via SFTP"
|
||||
log "Starting Restic backup"
|
||||
|
||||
lftp "sftp://${AppUser}:${AppPassword}@webbuilder.ackcentuate.com" -e \
|
||||
"cd /home/Backups; put /home/Backups/${HostPrefix}${ArchiveFile}; bye"
|
||||
# Capture restic's JSON output in a temporary file. Errors and warnings
|
||||
# still go to the run log via stderr.
|
||||
RESTIC_OUTPUT="$(mktemp /tmp/restic-backup-XXXXXX.json)"
|
||||
|
||||
log "SFTP upload completed successfully"
|
||||
restic backup "/home/${AppUser}/" \
|
||||
--json \
|
||||
--tag apache \
|
||||
--tag website \
|
||||
--tag "$AppName" \
|
||||
--tag "$HOSTNAME_FQDN" \
|
||||
> "$RESTIC_OUTPUT" 2>> "$RUNLOG"
|
||||
|
||||
log "Restic backup completed successfully"
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Log Restic Snapshot Details
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
summary=$(jq -cR 'fromjson? | select(.message_type == "summary")' \
|
||||
"$RESTIC_OUTPUT" | tail -n 1)
|
||||
|
||||
cleanup_restic_output
|
||||
|
||||
if [[ -n "$summary" ]]; then
|
||||
log_snapshot_info "$summary"
|
||||
else
|
||||
|
||||
log "Loading Restic environment"
|
||||
|
||||
source /root/.restic-env
|
||||
|
||||
log "Starting Restic backup"
|
||||
|
||||
restic backup "/home/${AppUser}/" \
|
||||
--tag apache \
|
||||
--tag website \
|
||||
--tag "$AppName" \
|
||||
--tag "$HOSTNAME_FQDN" \
|
||||
>> "$RUNLOG" 2>&1
|
||||
|
||||
log "Restic backup completed successfully"
|
||||
|
||||
log "WARNING: Restic returned no snapshot summary for ${AppName}"
|
||||
fi
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
+117
-48
@@ -3,8 +3,8 @@
|
||||
### Docker Stack Backup Script
|
||||
### Created by Charlie Silverman - ackitsme3@gmail.com
|
||||
###############################################################################
|
||||
### Version: 2.1.1
|
||||
### Updated: 2026-10-18
|
||||
### Version: 2.1.4
|
||||
### Updated: 2026-10-26
|
||||
### Purpose: Backs up all docker stacks in a given directory to Restic bucket
|
||||
###############################################################################
|
||||
### Requirements:
|
||||
@@ -38,6 +38,15 @@
|
||||
# Added automatic RUN_LOG cleanup during failure and interruption.
|
||||
# Improved housekeeping to prevent orphaned staging data after
|
||||
# unexpected termination.
|
||||
# 2.1.2 - Added EMAIL_TO and EMAIL_FROM variables for reports.
|
||||
# 2.1.3 - Added DOCKERUSER variable for account to use for launching contaners
|
||||
# 2.1.4 - Restic snapshot details (ID, parent, file/dir counts, data added,
|
||||
# total size, duration) are now written to the logs.
|
||||
# restic backup now runs with --json; snapshot size is read from the
|
||||
# backup summary instead of parsing 'restic snapshots' table output,
|
||||
# so it always refers to this run's snapshot.
|
||||
# Removed dependency on bc (awk used for arithmetic).
|
||||
# Added total new data uploaded to end of log.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
@@ -45,10 +54,13 @@ set -Eeuo pipefail
|
||||
# Configuration
|
||||
###############################################################################
|
||||
|
||||
COMPOSE_ROOT="/home/ackitsme/docker"
|
||||
LOG_DIR="/home/ackitsme/docker/backuplogs"
|
||||
COMPOSE_ROOT="/home/docker"
|
||||
LOG_DIR="/home/docker/backuplogs"
|
||||
mkdir -p "${LOG_DIR}"
|
||||
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
|
||||
EMAIL_TO="charlie@kiscc.com"
|
||||
EMAIL_FROM="kisprodweb.dockerbackup@kiscc.com"
|
||||
DOCKERUSER="csilverman"
|
||||
|
||||
# Restic repository configuration
|
||||
export RESTIC_ENV_FILE="/root/.restic-env"
|
||||
@@ -61,6 +73,7 @@ TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
|
||||
RUN_LOG="${LOG_DIR}/docker-backup_run_${TIMESTAMP}.log"
|
||||
CURRENT_STAGING_DIR=""
|
||||
RESTIC_OUTPUT=""
|
||||
|
||||
touch "${RUN_LOG}"
|
||||
|
||||
@@ -68,7 +81,7 @@ touch "${RUN_LOG}"
|
||||
# Verify Requirements
|
||||
###############################################################################
|
||||
|
||||
for cmd in docker restic tar gzip jq; do
|
||||
for cmd in docker restic tar gzip jq awk; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "Missing required command: $cmd"
|
||||
exit 1
|
||||
@@ -91,8 +104,8 @@ fi
|
||||
|
||||
## Log to screen and log file
|
||||
log() {
|
||||
local message
|
||||
message="[$(date '+%F %T')] $*"
|
||||
local message
|
||||
message="[$(date '+%F %T')] $*"
|
||||
|
||||
echo "$message"
|
||||
|
||||
@@ -100,6 +113,60 @@ log() {
|
||||
echo "$message" >> "$RUN_LOG"
|
||||
}
|
||||
|
||||
## Convert a byte count to a human-readable size (e.g. 1.23GiB)
|
||||
human_bytes() {
|
||||
local bytes="${1:-0}"
|
||||
|
||||
if command -v numfmt >/dev/null 2>&1; then
|
||||
numfmt --to=iec-i --suffix=B --format="%.2f" "$bytes" 2>/dev/null \
|
||||
|| echo "${bytes} B"
|
||||
else
|
||||
echo "${bytes} B"
|
||||
fi
|
||||
}
|
||||
|
||||
## Write Restic snapshot details from a backup summary (JSON) to the log
|
||||
log_snapshot_info() {
|
||||
local summary="$1"
|
||||
|
||||
local snapshot_id parent_id
|
||||
local files_new files_changed files_unmodified
|
||||
local dirs_new dirs_changed dirs_unmodified
|
||||
local data_added data_added_packed
|
||||
local total_files total_bytes total_duration
|
||||
|
||||
snapshot_id=$(jq -r '.snapshot_id // "none"' <<< "$summary")
|
||||
parent_id=$(restic snapshots "$snapshot_id" --json 2>/dev/null \
|
||||
| jq -r '.[0].parent // "none"' 2>/dev/null || echo "unknown")
|
||||
|
||||
files_new=$(jq -r '.files_new // 0' <<< "$summary")
|
||||
files_changed=$(jq -r '.files_changed // 0' <<< "$summary")
|
||||
files_unmodified=$(jq -r '.files_unmodified // 0' <<< "$summary")
|
||||
dirs_new=$(jq -r '.dirs_new // 0' <<< "$summary")
|
||||
dirs_changed=$(jq -r '.dirs_changed // 0' <<< "$summary")
|
||||
dirs_unmodified=$(jq -r '.dirs_unmodified // 0' <<< "$summary")
|
||||
data_added=$(jq -r '.data_added // 0' <<< "$summary")
|
||||
data_added_packed=$(jq -r '.data_added_packed // empty' <<< "$summary")
|
||||
total_files=$(jq -r '.total_files_processed // 0' <<< "$summary")
|
||||
total_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
||||
total_duration=$(jq -r '.total_duration // 0' <<< "$summary")
|
||||
|
||||
log "---------------- Restic Snapshot -----------------"
|
||||
log "Snapshot ID: ${snapshot_id:0:8} (${snapshot_id})"
|
||||
log "Parent snapshot: ${parent_id:0:8}"
|
||||
log "Tags: docker, compose, ${project_name}, ${HOSTNAME_SHORT}"
|
||||
log "Files: ${files_new} new, ${files_changed} changed, ${files_unmodified} unmodified"
|
||||
log "Directories: ${dirs_new} new, ${dirs_changed} changed, ${dirs_unmodified} unmodified"
|
||||
if [[ -n "$data_added_packed" ]]; then
|
||||
log "Data added: $(human_bytes "$data_added") ($(human_bytes "$data_added_packed") stored)"
|
||||
else
|
||||
log "Data added: $(human_bytes "$data_added")"
|
||||
fi
|
||||
log "Snapshot size: $(human_bytes "$total_bytes") in ${total_files} files"
|
||||
log "Duration: $(awk -v d="$total_duration" 'BEGIN{printf "%.1f", d}') seconds"
|
||||
log "--------------------------------------------------"
|
||||
}
|
||||
|
||||
## Check if specified project is in the exception list
|
||||
should_skip() {
|
||||
local project="$1"
|
||||
@@ -123,7 +190,7 @@ restart_stack() {
|
||||
|
||||
log "Restarting stack: ${compose_dir}"
|
||||
|
||||
su - ackitsme -c \
|
||||
su - $DOCKERUSER -c \
|
||||
"cd '${compose_dir}' && docker compose up -d" \
|
||||
|| log "WARNING: Failed to restart stack ${compose_dir}"
|
||||
}
|
||||
@@ -132,21 +199,7 @@ restart_stack() {
|
||||
send_email_log() {
|
||||
local subject="$1"
|
||||
|
||||
if command -v mailx >/dev/null 2>&1; then
|
||||
|
||||
mailx -s "$subject" \
|
||||
ackitsme3@gmail.com < "$RUN_LOG"
|
||||
|
||||
elif command -v mail >/dev/null 2>&1; then
|
||||
|
||||
mail -s "$subject" \
|
||||
ackitsme3@gmail.com < "$RUN_LOG"
|
||||
|
||||
else
|
||||
|
||||
log "WARNING: mail/mailx command not found. Email not sent."
|
||||
|
||||
fi
|
||||
mailx -s "$subject" -r "$EMAIL_FROM" "$EMAIL_TO" < "$RUN_LOG"
|
||||
}
|
||||
|
||||
## Cleanup Function
|
||||
@@ -161,6 +214,10 @@ cleanup_staging() {
|
||||
log "WARNING: Failed to remove ${CURRENT_STAGING_DIR}"
|
||||
|
||||
fi
|
||||
|
||||
if [[ -n "${RESTIC_OUTPUT:-}" ]]; then
|
||||
rm -f "${RESTIC_OUTPUT}"
|
||||
fi
|
||||
}
|
||||
|
||||
## Error Handler
|
||||
@@ -185,7 +242,7 @@ error_handler() {
|
||||
|
||||
## Interrupt Handler
|
||||
interrupt_handler() {
|
||||
trap - SIGINT SIGTERM
|
||||
trap - SIGINT SIGTERM
|
||||
|
||||
log "ERROR: Backup interrupted"
|
||||
|
||||
@@ -194,7 +251,7 @@ interrupt_handler() {
|
||||
send_email_log \
|
||||
"INTERRUPTED: Docker Backup (${HOSTNAME_SHORT})"
|
||||
|
||||
rm -f "${RUN_LOG}"
|
||||
rm -f "${RUN_LOG}"
|
||||
|
||||
exit 1
|
||||
}
|
||||
@@ -240,6 +297,7 @@ fi
|
||||
###############################################################################
|
||||
|
||||
total_size=0.00
|
||||
total_added=0
|
||||
|
||||
for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
|
||||
@@ -257,7 +315,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
log "=================================================="
|
||||
|
||||
STAGING_DIR="/home/Backups/docker-backup-${project_name}"
|
||||
CURRENT_STAGING_DIR="${STAGING_DIR}"
|
||||
CURRENT_STAGING_DIR="${STAGING_DIR}"
|
||||
|
||||
mkdir -p \
|
||||
"${STAGING_DIR}/images" \
|
||||
@@ -327,8 +385,8 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
|
||||
log "Saving image: $image"
|
||||
|
||||
docker save "$image" \
|
||||
> "${STAGING_DIR}/images/${safe_name}.tar"
|
||||
docker save "$image" \
|
||||
> "${STAGING_DIR}/images/${safe_name}.tar"
|
||||
fi
|
||||
|
||||
done < "${STAGING_DIR}/metadata/images.list"
|
||||
@@ -380,43 +438,53 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
|
||||
log "Uploading project backup to Restic..."
|
||||
|
||||
# Capture restic's JSON output (outside the staging dir so it is
|
||||
# not part of the backup). Errors/warnings still go to stderr.
|
||||
RESTIC_OUTPUT="$(mktemp /tmp/restic-backup-XXXXXX.json)"
|
||||
|
||||
restic backup \
|
||||
"${STAGING_DIR}" \
|
||||
--json \
|
||||
--tag docker \
|
||||
--tag compose \
|
||||
--tag "${project_name}" \
|
||||
--tag "${HOSTNAME_SHORT}"
|
||||
--tag "${HOSTNAME_SHORT}" \
|
||||
> "${RESTIC_OUTPUT}"
|
||||
|
||||
###################################################
|
||||
# Get backup size
|
||||
# Log snapshot details and get backup size
|
||||
###################################################
|
||||
# lastsnap=$(restic ls latest)
|
||||
# lastsnapid=$(echo "$lastsnap" | awk '{print $2}')
|
||||
lastsnapid=$(restic snapshots --json | jq -r '.[-1].short_id')
|
||||
# lastsnapid=$(restic snapshots latest --compact | awk 'NR==3 {print $1}')
|
||||
snapdata=$(restic snapshots "$lastsnapid")
|
||||
snapnum=$(echo $snapdata | awk '{print $14}')
|
||||
snapsu=$(echo $snapdata | awk '{print $15}')
|
||||
case "$snapsu" in
|
||||
"TiB")
|
||||
snapnum=$(echo "scale=2;$snapnum * 1024" | bc)
|
||||
;;
|
||||
"MiB")
|
||||
snapnum=$(echo "scale=2;$snapnum / 1024" | bc)
|
||||
;;
|
||||
esac
|
||||
snapnum=$(printf "%.2f\n" "$snapnum")
|
||||
|
||||
summary=$(jq -cR 'fromjson? | select(.message_type == "summary")' \
|
||||
"${RESTIC_OUTPUT}" | tail -n 1)
|
||||
|
||||
rm -f "${RESTIC_OUTPUT}"
|
||||
RESTIC_OUTPUT=""
|
||||
|
||||
if [[ -n "$summary" ]]; then
|
||||
log_snapshot_info "$summary"
|
||||
|
||||
snap_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
||||
added_bytes=$(jq -r '.data_added // 0' <<< "$summary")
|
||||
else
|
||||
log "WARNING: Restic returned no snapshot summary for ${project_name}"
|
||||
snap_bytes=0
|
||||
added_bytes=0
|
||||
fi
|
||||
|
||||
snapnum=$(awk -v b="$snap_bytes" 'BEGIN{printf "%.2f", b / 1073741824}')
|
||||
|
||||
###################################################
|
||||
# Cleanup Local Backup
|
||||
###################################################
|
||||
|
||||
rm -rf "${STAGING_DIR}"
|
||||
CURRENT_STAGING_DIR=""
|
||||
CURRENT_STAGING_DIR=""
|
||||
|
||||
log "Project complete: ${project_name} ${snapnum} GiB"
|
||||
log ""
|
||||
total_size=$(bc <<< "$total_size + $snapnum")
|
||||
total_size=$(awk -v a="$total_size" -v b="$snapnum" 'BEGIN{printf "%.2f", a + b}')
|
||||
total_added=$(awk -v a="$total_added" -v b="$added_bytes" 'BEGIN{printf "%.0f", a + b}')
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
@@ -433,6 +501,7 @@ restic forget \
|
||||
|
||||
log "All compose projects backed up successfully."
|
||||
log "Total Backup Size: $total_size GiB"
|
||||
log "Total New Data Uploaded: $(human_bytes "$total_added")"
|
||||
|
||||
send_email_log \
|
||||
"SUCCESS: Docker Backup (${HOSTNAME_SHORT})" || \
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Script Name: openwrtupload.sh
|
||||
# Description:
|
||||
# Backs up OpenWrt configuration archives to a Backblaze B2 repository
|
||||
# using Restic. Each .gz archive found in the source directory is
|
||||
# extracted to a staging directory, backed up with Restic, and then
|
||||
# removed along with the staging directory.
|
||||
#
|
||||
# Usage:
|
||||
# openwrtupload.sh
|
||||
#
|
||||
# Author: Charlie Silverman
|
||||
# Version: 20261031-01
|
||||
#
|
||||
# Requirements:
|
||||
# restic, jq, awk, tar, numfmt (optional, for human-readable sizes).
|
||||
# Restic environment file: /root/.restic-env
|
||||
#
|
||||
# Versioning:
|
||||
# Versions use the format YYYYMMDD-xx, where YYYYMMDD is the release date
|
||||
# and xx is the release number for that day, starting at 01.
|
||||
#
|
||||
# Revision History:
|
||||
# 20261031-01 - Added script header and YYYYMMDD-xx versioning.
|
||||
# Added strict error handling with error/interrupt logging;
|
||||
# an archive is no longer deleted if its backup fails.
|
||||
# Staging directory is cleaned up on failure or interrupt.
|
||||
# Restic snapshot details (ID, parent, tags, file/dir counts,
|
||||
# data added, snapshot size, duration) are now written to the
|
||||
# log. restic backup now runs with --json and the details are
|
||||
# read from its backup summary. Added totals at end of run.
|
||||
# Added required-command check.
|
||||
#
|
||||
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_VERSION="20261031-01"
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Configuration
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
LOG_FILE="/home/glibackup/openwrt-backup.log"
|
||||
SOURCE_DIR="/home/glibackup/openwrt"
|
||||
STAGING_DIR="/home/Backups/openwrt"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
HOSTNAME_FQDN="$(hostname -f)"
|
||||
|
||||
RESTIC_TAGS=(openwrt backup)
|
||||
|
||||
# Restic repository configuration
|
||||
export RESTIC_REPOSITORY="s3:https://s3.us-west-004.backblazeb2.com/ackitsbucket/openwrt"
|
||||
export RESTIC_ENV_FILE="/root/.restic-env"
|
||||
|
||||
RESTIC_OUTPUT=""
|
||||
STAGING_IN_USE=""
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Logging Function
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
log() {
|
||||
|
||||
local message
|
||||
|
||||
message="[$(date '+%F %T')] $*"
|
||||
|
||||
echo "$message"
|
||||
echo "$message" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Convert a byte count to a human-readable size (e.g. 1.23GiB)
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
human_bytes() {
|
||||
|
||||
local bytes="${1:-0}"
|
||||
|
||||
if command -v numfmt >/dev/null 2>&1; then
|
||||
numfmt --to=iec-i --suffix=B --format="%.2f" "$bytes" 2>/dev/null \
|
||||
|| echo "${bytes} B"
|
||||
else
|
||||
echo "${bytes} B"
|
||||
fi
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Write Restic snapshot details from a backup summary (JSON) to the log
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
log_snapshot_info() {
|
||||
|
||||
local summary="$1"
|
||||
|
||||
local snapshot_id parent_id
|
||||
local files_new files_changed files_unmodified
|
||||
local dirs_new dirs_changed dirs_unmodified
|
||||
local data_added data_added_packed
|
||||
local total_files total_bytes total_duration
|
||||
local tags
|
||||
|
||||
snapshot_id=$(jq -r '.snapshot_id // "none"' <<< "$summary")
|
||||
parent_id=$(restic snapshots "$snapshot_id" --json 2>/dev/null \
|
||||
| jq -r '.[0].parent // "none"' 2>/dev/null || echo "unknown")
|
||||
|
||||
files_new=$(jq -r '.files_new // 0' <<< "$summary")
|
||||
files_changed=$(jq -r '.files_changed // 0' <<< "$summary")
|
||||
files_unmodified=$(jq -r '.files_unmodified // 0' <<< "$summary")
|
||||
dirs_new=$(jq -r '.dirs_new // 0' <<< "$summary")
|
||||
dirs_changed=$(jq -r '.dirs_changed // 0' <<< "$summary")
|
||||
dirs_unmodified=$(jq -r '.dirs_unmodified // 0' <<< "$summary")
|
||||
data_added=$(jq -r '.data_added // 0' <<< "$summary")
|
||||
data_added_packed=$(jq -r '.data_added_packed // empty' <<< "$summary")
|
||||
total_files=$(jq -r '.total_files_processed // 0' <<< "$summary")
|
||||
total_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
||||
total_duration=$(jq -r '.total_duration // 0' <<< "$summary")
|
||||
|
||||
tags=$(IFS=,; echo "${RESTIC_TAGS[*]}" | sed 's/,/, /g')
|
||||
|
||||
log "---------------- Restic Snapshot -----------------"
|
||||
log "Snapshot ID: ${snapshot_id:0:8} (${snapshot_id})"
|
||||
log "Parent snapshot: ${parent_id:0:8}"
|
||||
log "Tags: ${tags}"
|
||||
log "Files: ${files_new} new, ${files_changed} changed, ${files_unmodified} unmodified"
|
||||
log "Directories: ${dirs_new} new, ${dirs_changed} changed, ${dirs_unmodified} unmodified"
|
||||
if [[ -n "$data_added_packed" ]]; then
|
||||
log "Data added: $(human_bytes "$data_added") ($(human_bytes "$data_added_packed") stored)"
|
||||
else
|
||||
log "Data added: $(human_bytes "$data_added")"
|
||||
fi
|
||||
log "Snapshot size: $(human_bytes "$total_bytes") in ${total_files} files"
|
||||
log "Duration: $(awk -v d="$total_duration" 'BEGIN{printf "%.1f", d}') seconds"
|
||||
log "--------------------------------------------------"
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Cleanup Function
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
cleanup() {
|
||||
|
||||
if [[ -n "${STAGING_IN_USE:-}" && -d "$STAGING_DIR" ]]; then
|
||||
log "Cleaning up staging directory ${STAGING_DIR}"
|
||||
rm -rf "$STAGING_DIR" || log "WARNING: Failed to remove ${STAGING_DIR}"
|
||||
STAGING_IN_USE=""
|
||||
fi
|
||||
|
||||
if [[ -n "${RESTIC_OUTPUT:-}" ]]; then
|
||||
rm -f "$RESTIC_OUTPUT"
|
||||
RESTIC_OUTPUT=""
|
||||
fi
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Error Handling
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
error_handler() {
|
||||
|
||||
local exit_code=$?
|
||||
local line_number=$1
|
||||
|
||||
trap - ERR
|
||||
|
||||
log "ERROR: Backup failed on line ${line_number} with exit code ${exit_code}"
|
||||
|
||||
cleanup
|
||||
|
||||
log "----------------------------------------------------"
|
||||
|
||||
exit "${exit_code}"
|
||||
}
|
||||
|
||||
interrupt_handler() {
|
||||
|
||||
trap - SIGINT SIGTERM
|
||||
|
||||
log "ERROR: Backup interrupted by signal"
|
||||
|
||||
cleanup
|
||||
|
||||
log "----------------------------------------------------"
|
||||
|
||||
exit 1
|
||||
}
|
||||
|
||||
trap 'error_handler $LINENO' ERR
|
||||
trap interrupt_handler SIGINT SIGTERM
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Startup Logging
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
log "===================================================="
|
||||
log "OpenWrt Backup Starting (openwrtupload.sh v${SCRIPT_VERSION})"
|
||||
log "Host: ${HOSTNAME_FQDN}"
|
||||
log "Source Directory: ${SOURCE_DIR}"
|
||||
log "Log File: ${LOG_FILE}"
|
||||
log "===================================================="
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Verify Requirements
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
for cmd in restic jq awk tar; do
|
||||
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
log "ERROR: Missing required command: $cmd"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Load Restic Environment
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "$RESTIC_ENV_FILE"
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Locate Archives
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
shopt -s nullglob
|
||||
ARCHIVES=("$SOURCE_DIR"/*.gz)
|
||||
shopt -u nullglob
|
||||
|
||||
if [[ ${#ARCHIVES[@]} -eq 0 ]]; then
|
||||
log "No backup files found."
|
||||
log "----------------------------------------------------"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Found ${#ARCHIVES[@]} archive(s) to back up"
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Process Each Archive
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
TAG_ARGS=()
|
||||
for tag in "${RESTIC_TAGS[@]}"; do
|
||||
TAG_ARGS+=(--tag "$tag")
|
||||
done
|
||||
|
||||
total_bytes_all=0
|
||||
total_added_all=0
|
||||
archive_count=0
|
||||
|
||||
for FILE in "${ARCHIVES[@]}"; do
|
||||
|
||||
log "Backing up archive ${FILE}..."
|
||||
|
||||
# Start from an empty staging directory
|
||||
rm -rf "$STAGING_DIR"
|
||||
mkdir -p "$STAGING_DIR"
|
||||
STAGING_IN_USE="yes"
|
||||
|
||||
log "Extracting archive ${FILE} to ${STAGING_DIR}"
|
||||
|
||||
tar xzf "$FILE" -C "$STAGING_DIR"
|
||||
|
||||
log "Backing up ${STAGING_DIR} with restic"
|
||||
|
||||
# Capture restic's JSON output in a temporary file. Errors and
|
||||
# warnings still go to the log via stderr.
|
||||
RESTIC_OUTPUT="$(mktemp /tmp/restic-backup-XXXXXX.json)"
|
||||
|
||||
restic backup \
|
||||
"$STAGING_DIR" \
|
||||
--json \
|
||||
"${TAG_ARGS[@]}" \
|
||||
> "$RESTIC_OUTPUT" 2>> "$LOG_FILE"
|
||||
|
||||
summary=$(jq -cR 'fromjson? | select(.message_type == "summary")' \
|
||||
"$RESTIC_OUTPUT" | tail -n 1)
|
||||
|
||||
rm -f "$RESTIC_OUTPUT"
|
||||
RESTIC_OUTPUT=""
|
||||
|
||||
if [[ -n "$summary" ]]; then
|
||||
log_snapshot_info "$summary"
|
||||
snap_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
||||
added_bytes=$(jq -r '.data_added // 0' <<< "$summary")
|
||||
else
|
||||
log "WARNING: Restic returned no snapshot summary for ${FILE}"
|
||||
snap_bytes=0
|
||||
added_bytes=0
|
||||
fi
|
||||
|
||||
total_bytes_all=$(awk -v a="$total_bytes_all" -v b="$snap_bytes" 'BEGIN{printf "%.0f", a + b}')
|
||||
total_added_all=$(awk -v a="$total_added_all" -v b="$added_bytes" 'BEGIN{printf "%.0f", a + b}')
|
||||
archive_count=$((archive_count + 1))
|
||||
|
||||
log "Cleaning up staging directory ${STAGING_DIR}"
|
||||
rm -rf "$STAGING_DIR"
|
||||
STAGING_IN_USE=""
|
||||
|
||||
log "Backup Complete."
|
||||
|
||||
log "Removing archive file ${FILE}"
|
||||
rm -f "$FILE"
|
||||
|
||||
log "----------------------------------------------------"
|
||||
|
||||
done
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Completion
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
log "All archives backed up successfully."
|
||||
log "Archives Processed: ${archive_count}"
|
||||
log "Total Backup Size: $(human_bytes "$total_bytes_all")"
|
||||
log "Total New Data Uploaded: $(human_bytes "$total_added_all")"
|
||||
log "===================================================="
|
||||
|
||||
exit 0
|
||||
+342
-189
@@ -1,223 +1,376 @@
|
||||
#!/usr/bin/env bash
|
||||
SCRIPTNAME="$(basename "$0")"
|
||||
VERSION=3.5.0
|
||||
FILEDATE="2026-07-26"
|
||||
|
||||
echo -e "\033[0m==============================================="
|
||||
echo -e " $SCRIPTNAME - rsync progress monitor"
|
||||
echo -e " Written by Charlie Silverman"
|
||||
echo -e " Version: $VERSION"
|
||||
echo -e " Release Date: $FILEDATE"
|
||||
echo -e "==============================================="
|
||||
###############################################################################
|
||||
# Script Name : RsyncMonitor.sh
|
||||
# Author : Charlie Silverman
|
||||
# Version : 4.1.0
|
||||
# Date : 2026-10-18
|
||||
#
|
||||
# Description:
|
||||
# Monitor rsync copy progress by comparing source and destination sizes.
|
||||
# Calculates completion percentage, transfer rate, and estimated time
|
||||
# remaining.
|
||||
#
|
||||
# Revision History
|
||||
# -----------------------------------------------------------------------------
|
||||
# v4.1.0 - 2026-10-18
|
||||
# - Converted to getopts
|
||||
# - Added configurable logfile (-l)
|
||||
# - Added required option validation
|
||||
# - Improved error handling
|
||||
#
|
||||
# v4.0.0 - 2026-10-18
|
||||
# - Added logging framework
|
||||
# - Added trap handlers
|
||||
# - Added standardized help
|
||||
# - Fixed syntax issues
|
||||
#
|
||||
# v3.5.0 - 2026-07-26
|
||||
# - Original release
|
||||
###############################################################################
|
||||
|
||||
SCRIPT_NAME="$(basename "$0")"
|
||||
SCRIPT_VERSION="4.1.0"
|
||||
SCRIPT_DATE="2026-10-18"
|
||||
|
||||
set -o pipefail
|
||||
|
||||
###############################################################################
|
||||
# Defaults
|
||||
###############################################################################
|
||||
|
||||
SRCFOLDER=""
|
||||
DSTFOLDER=""
|
||||
SLEEP=60
|
||||
HELP=false
|
||||
LOGFILE="/tmp/RsyncMonitor.log"
|
||||
|
||||
###############################################################################
|
||||
# Logging Functions
|
||||
###############################################################################
|
||||
|
||||
log_info() {
|
||||
local MESSAGE="$1"
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO ] $MESSAGE" | tee -a "$LOGFILE"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
local MESSAGE="$1"
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [WARN ] $MESSAGE" | tee -a "$LOGFILE"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
local MESSAGE="$1"
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $MESSAGE" | tee -a "$LOGFILE" >&2
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Error Handling
|
||||
###############################################################################
|
||||
|
||||
error_handler() {
|
||||
local LINE="$1"
|
||||
log_error "Unexpected error on line ${LINE}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
interrupt_handler() {
|
||||
log_warn "Execution interrupted."
|
||||
exit 130
|
||||
}
|
||||
|
||||
trap 'error_handler $LINENO' ERR
|
||||
trap interrupt_handler SIGINT SIGTERM
|
||||
|
||||
###############################################################################
|
||||
# Banner
|
||||
###############################################################################
|
||||
|
||||
show_banner() {
|
||||
|
||||
cat << EOF
|
||||
|
||||
==================================================
|
||||
${SCRIPT_NAME}
|
||||
rsync Progress Monitor
|
||||
|
||||
Author : Charlie Silverman
|
||||
Version : ${SCRIPT_VERSION}
|
||||
Date : ${SCRIPT_DATE}
|
||||
==================================================
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Help
|
||||
###############################################################################
|
||||
|
||||
show_help() {
|
||||
|
||||
cat << EOF
|
||||
|
||||
NAME
|
||||
${SCRIPT_NAME}
|
||||
|
||||
SYNOPSIS
|
||||
|
||||
${SCRIPT_NAME} -s <source> -d <destination> [options]
|
||||
|
||||
OPTIONS
|
||||
|
||||
-s <path>
|
||||
Source folder to monitor.
|
||||
|
||||
-d <path>
|
||||
Destination folder to monitor.
|
||||
|
||||
-t <seconds>
|
||||
Sleep interval between scans.
|
||||
|
||||
Default: 60
|
||||
|
||||
-l <file>
|
||||
Log file path.
|
||||
|
||||
Default:
|
||||
/tmp/RsyncMonitor.log
|
||||
|
||||
-h
|
||||
Display help.
|
||||
|
||||
EXAMPLES
|
||||
|
||||
${SCRIPT_NAME} \\
|
||||
-s /home/olduser \\
|
||||
-d /home/newuser
|
||||
|
||||
${SCRIPT_NAME} \\
|
||||
-s /source \
|
||||
-d /destination \
|
||||
-t 300 \
|
||||
-l /var/log/RsyncMonitor.log
|
||||
|
||||
EOF
|
||||
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Utilities
|
||||
###############################################################################
|
||||
|
||||
get_directory_size_mb() {
|
||||
|
||||
local DIR="$1"
|
||||
local MOUNT
|
||||
|
||||
if [[ "${DIR: -1}" == "/" ]]; then
|
||||
MOUNT=$(df "${DIR%?}" | awk 'NR==2 {print $6}')
|
||||
else
|
||||
MOUNT=$(df "$DIR" | awk 'NR==2 {print $6}')
|
||||
fi
|
||||
|
||||
if [[ "$MOUNT" != "$DIR" ]]; then
|
||||
du -sm "$DIR" | awk '{print $1}'
|
||||
else
|
||||
local USED_KB
|
||||
USED_KB=$(df "$DIR" | awk 'NR==2 {print $3}')
|
||||
echo $(( USED_KB / 1024 ))
|
||||
fi
|
||||
}
|
||||
|
||||
format_size() {
|
||||
|
||||
local SIZE_MB="$1"
|
||||
|
||||
if (( SIZE_MB >= 1024 )); then
|
||||
local SIZE_GB
|
||||
SIZE_GB=$(echo "scale=3;$SIZE_MB/1024" | bc)
|
||||
printf "%'.3fGB" "$SIZE_GB"
|
||||
else
|
||||
echo "${SIZE_MB}MB"
|
||||
fi
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Option Processing
|
||||
###############################################################################
|
||||
|
||||
while getopts ":s:d:t:l:h" opt; do
|
||||
|
||||
case "${opt}" in
|
||||
|
||||
s)
|
||||
SRCFOLDER="$OPTARG"
|
||||
;;
|
||||
|
||||
d)
|
||||
DSTFOLDER="$OPTARG"
|
||||
;;
|
||||
|
||||
t)
|
||||
SLEEP="$OPTARG"
|
||||
;;
|
||||
|
||||
l)
|
||||
LOGFILE="$OPTARG"
|
||||
;;
|
||||
|
||||
h)
|
||||
show_help
|
||||
exit 0
|
||||
;;
|
||||
|
||||
:)
|
||||
echo "Missing argument for -${OPTARG}"
|
||||
exit 1
|
||||
;;
|
||||
|
||||
\?)
|
||||
echo "Invalid option: -${OPTARG}"
|
||||
exit 1
|
||||
;;
|
||||
|
||||
esac
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
echo $1
|
||||
IFS="=" read -r command value <<< "$1"
|
||||
echo "Command: $command"
|
||||
echo "Value: $value"
|
||||
echo "Value length: ${#value}"
|
||||
case "$command" in
|
||||
--source)
|
||||
if [ "${#value})" -gt 0 ]; then
|
||||
SRCFOLDER="$value"
|
||||
shift
|
||||
else
|
||||
SRCFOLDER="$2"
|
||||
shift 2
|
||||
fi
|
||||
;;
|
||||
--destination)
|
||||
if [ "${#value})" -gt 0 ]; then
|
||||
DSTFOLDER="$value"
|
||||
shift
|
||||
else
|
||||
DSTFOLDER="$2"
|
||||
shift 2
|
||||
fi
|
||||
;;
|
||||
--sleep)
|
||||
if [ "${#value})" -gt 0 ]; then
|
||||
SLEEP="$value"
|
||||
shift
|
||||
else
|
||||
SLEEP="$2"
|
||||
shift 2
|
||||
fi
|
||||
;;
|
||||
--help)
|
||||
HELP=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unknown argument: $1"
|
||||
HELP=true
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
# Startup
|
||||
###############################################################################
|
||||
|
||||
show_banner
|
||||
|
||||
touch "$LOGFILE" 2>/dev/null || {
|
||||
echo "Unable to write logfile: $LOGFILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Validation
|
||||
###############################################################################
|
||||
|
||||
ERROR=0
|
||||
|
||||
if [[ "$HELP" != "true" ]]; then
|
||||
# Check if source folder exists
|
||||
if [[ ! -d "${SRCFOLDER}" ]]; then
|
||||
echo -e "\n\033[0;31mE\e[4mRROR: Source Folder $SRCFOLDER does not exist.\e[0m"
|
||||
ERROR=1
|
||||
fi
|
||||
# Check if destination folder exists
|
||||
if [[ ! -d "${DSTFOLDER}" ]]; then
|
||||
echo -e "\n\033[0;31mE\e[4mRROR: Destination Folder $DSTFOLDER does not exist.\e[0m"
|
||||
ERROR=1
|
||||
fi
|
||||
# Verify that a sleep value was supplied
|
||||
if [ -z "${SLEEP}" ]; then
|
||||
echo -e "\n\033[0;31mE\e[4mRROR: Sleep delay not specified.\e[0m"
|
||||
ERROR=1
|
||||
else
|
||||
# Verify that the supplied sleep value is actually a Number
|
||||
re='^[0-9]+$'
|
||||
if ! [[ $SLEEP =~ $re ]]; then
|
||||
echo -e "\n\033[0;31m\e[4mERROR: Sleep delay invalid.\e[0m"
|
||||
ERROR=1
|
||||
fi
|
||||
fi
|
||||
if [[ -z "$SRCFOLDER" ]]; then
|
||||
log_error "Source folder not specified."
|
||||
ERROR=1
|
||||
fi
|
||||
|
||||
# Help display
|
||||
if $HELP; then
|
||||
cat << EOF
|
||||
|
||||
SYNOPSIS
|
||||
./$SCRIPTNAME --source <SRCFOLDER> --destination <DSTFOLDER> --sleep <SECONDS> [OPTIONS]"
|
||||
|
||||
OPTIONS
|
||||
--source <SRCFOLDER>
|
||||
Full or relative path to the source rsync folder to monitor
|
||||
|
||||
--destination <DSTFOLDER>
|
||||
Full or relative path to the destination rsync folder to monitor
|
||||
|
||||
--sleep <SECONDS>
|
||||
The number of seconds to sleep between scanning passes.
|
||||
Default: 60
|
||||
|
||||
--help
|
||||
Display this help message and exit.
|
||||
|
||||
EXAMPLE
|
||||
./$SCRIPTNAME --source /home/olduser --destination /home/newuser --sleep 300"
|
||||
EOF
|
||||
if [[ -z "$DSTFOLDER" ]]; then
|
||||
log_error "Destination folder not specified."
|
||||
ERROR=1
|
||||
fi
|
||||
|
||||
echo "Calculating size of $SRCFOLDER..."
|
||||
|
||||
# Getting the mount point that relates to the source path, stripping off
|
||||
# the trailing slash if found
|
||||
if [[ "${SRCFOLDER: -1}" = "/" ]]; then
|
||||
SRCMOUNT=$(df ${SRCFOLDER%?} | sed -n '2p' | awk '{print $6}')
|
||||
else
|
||||
SRCMOUNT=$(df $SRCFOLDER | sed -n '2p' | awk '{print $6}')
|
||||
if [[ ! -d "$SRCFOLDER" ]]; then
|
||||
log_error "Source folder does not exist: $SRCFOLDER"
|
||||
ERROR=1
|
||||
fi
|
||||
|
||||
# Determining the size of the source data. If the source path
|
||||
# matches a mount point from df, assuming that mount point data
|
||||
# to be accurate.
|
||||
if [ $SRCMOUNT != $SRCFOLDER ]; then
|
||||
MEGS=$(du -sm $SRCFOLDER |awk '{print $1}')
|
||||
else
|
||||
MEGS=$(df $SRCFOLDER | sed -n '2p' | awk '{print $3}')
|
||||
MEGS=$(($MEGS / 1024))
|
||||
fi
|
||||
MEGSDISPLAY="${MEGS}MB"
|
||||
MEGSGB=$(echo "scale=2;$MEGS/1024" | bc)
|
||||
if [ $(echo "$MEGSGB > 0" | bc) -ne 0 ]; then
|
||||
NUMFMT=$(printf "%'.3f\n" ${MEGSGB})
|
||||
MEGSDISPLAY="${NUMFMT}GB"
|
||||
if [[ ! -d "$DSTFOLDER" ]]; then
|
||||
log_error "Destination folder does not exist: $DSTFOLDER"
|
||||
ERROR=1
|
||||
fi
|
||||
|
||||
echo "Monitoring $DSTFOLDER..."
|
||||
|
||||
# Getting the mount point that relates to the destination path, stripping off
|
||||
# the trailing slash if found
|
||||
if [[ "${$DSTFOLDER: -1}" = "/" ]]; then
|
||||
DSTMOUNT=$(df ${DSTFOLDER%?} | sed -n '2p' | awk '{print $6}')
|
||||
else
|
||||
DSTMOUNT=$(df $DSTFOLDER | sed -n '2p' | awk '{print $6}')
|
||||
if ! [[ "$SLEEP" =~ ^[0-9]+$ ]]; then
|
||||
log_error "Sleep value must be numeric."
|
||||
ERROR=1
|
||||
fi
|
||||
|
||||
SECONDSLEFT=1
|
||||
if (( ERROR > 0 )); then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# Calculate Source Size
|
||||
###############################################################################
|
||||
|
||||
log_info "Calculating source size."
|
||||
|
||||
MEGS=$(get_directory_size_mb "$SRCFOLDER")
|
||||
MEGSGB=$(echo "scale=3;$MEGS/1024" | bc)
|
||||
MEGSDISPLAY=$(format_size "$MEGS")
|
||||
|
||||
log_info "Source size: $MEGSDISPLAY"
|
||||
|
||||
###############################################################################
|
||||
# Initialize Destination
|
||||
###############################################################################
|
||||
|
||||
FIRST=$(get_directory_size_mb "$DSTFOLDER")
|
||||
FIRSTGB=$(echo "scale=3;$FIRST/1024" | bc)
|
||||
FIRSTDISPLAY=$(format_size "$FIRST")
|
||||
|
||||
STARTTIME=$(date +%s)
|
||||
# Determining the current size of the destination data. If the
|
||||
# destination path matches a mount point from df, assuming that
|
||||
# mount point data to be accurate.
|
||||
if [ $DSTMOUNT != $DSTFOLDER ]; then
|
||||
FIRST=$(du -sm $DSTFOLDER |awk '{print $1}')
|
||||
START="$FIRST"
|
||||
|
||||
if (( $(echo "$MEGSGB > 0" | bc) )); then
|
||||
CURRENTPCT=$(echo "scale=0;($FIRSTGB/$MEGSGB)*100" | bc)
|
||||
else
|
||||
FIRST=$(df $DSTFOLDER | sed -n '2p' | awk '{print $3}')
|
||||
FIRST=$(($FIRST / 1024))
|
||||
CURRENTPCT=0
|
||||
fi
|
||||
|
||||
FIRSTDISPLAY="${FIRST}MB"
|
||||
FIRSTGB=$(echo "scale=2;$FIRST/1024" | bc)
|
||||
if [ $(echo "$FIRSTGB > 0" | bc) -ne 0 ]; then
|
||||
NUMFMT=$(printf "%'.3f\n" ${FIRSTGB})
|
||||
FIRSTDISPLAY="${NUMFMT}GB"
|
||||
fi
|
||||
|
||||
CURRENTPCT=$(echo "scale=2;($FIRSTGB/$MEGSGB)*100" | bc | cut -d . -f 1)
|
||||
|
||||
START=$FIRST
|
||||
CMDTIME=0
|
||||
CURRENTTIME=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo -e "[${CURRENTTIME}] \033[0;32m${FIRSTDISPLAY}\e[0m of \033[0;32m${MEGSDISPLAY}\e[0m (\033[0;32m${CURRENTPCT}%\e[0m)\tStarting\t\t| Sleeping ${SLEEP}s..."
|
||||
while (( $SECONDSLEFT > 0 )); do
|
||||
sleep $SLEEP
|
||||
|
||||
# Determining the current size of the destination data. If the
|
||||
# destination path matches a mount point from df, assuming that
|
||||
# mount point data to be accurate.
|
||||
if [ $DSTMOUNT != $DSTFOLDER ]; then
|
||||
END=$(du -sm $DSTFOLDER |awk '{print $1}')
|
||||
else
|
||||
END=$(df $DSTFOLDER | sed -n '2p' | awk '{print $3}')
|
||||
END=$(($END / 1024))
|
||||
fi
|
||||
echo -e "[${CURRENTTIME}] ${FIRSTDISPLAY} of ${MEGSDISPLAY} (${CURRENTPCT}%) Starting | Sleeping ${SLEEP}s..."
|
||||
|
||||
TIMENOW=$(date +%s)
|
||||
TIMEDIFF=$(($TIMENOW-$STARTTIME))
|
||||
DOWNLOADED=$(( $END-$START ))
|
||||
TOTALDOWNLOADED=$(( $END-$FIRST ))
|
||||
if [[ $DOWNLOADED -lt 0 ]]; then
|
||||
DOWNLOADED=0
|
||||
fi
|
||||
###############################################################################
|
||||
# Monitor Loop
|
||||
###############################################################################
|
||||
|
||||
ENDDISPLAY="${END}MB"
|
||||
ENDGB=$(echo "scale=2;$END/1024" | bc)
|
||||
if [[ $(echo "$END > 0" | bc) -ne 0 ]]; then
|
||||
NUMFMT=$(printf "%'.3f\n" ${ENDGB})
|
||||
ENDDISPLAY="${NUMFMT}GB"
|
||||
fi
|
||||
while true; do
|
||||
|
||||
CURRENTTIME=$(date '+%Y-%m0%d %H:%M:%S')/
|
||||
CURRENTPCT=$(echo "scale=2;($ENDGB/$MEGSGB)*100" | bc | cut -d . -f 1)
|
||||
sleep "$SLEEP"
|
||||
|
||||
if [[ "$DOWNLOADED" == "0" ]]; then
|
||||
echo -e "[${CURRENTTIME}] \033[0;32m${ENDDISPLAY}\e[0m of \033[0;32m${MEGSDISPLAY}\e[0m (\033[0;32m${CURRENTPCT%}\e[0m)\t| No change since last check | Sleeping ${THISSLEEP}s..."
|
||||
continue
|
||||
fi
|
||||
END=$(get_directory_size_mb "$DSTFOLDER")
|
||||
ENDDISPLAY=$(format_size "$END")
|
||||
ENDGB=$(echo "scale=3;$END/1024" | bc)
|
||||
|
||||
OVERALLMEGSPERSECOND=$(( $TOTALDOWNLOADED/$TIMEDIFF ))
|
||||
OVERALLSECONDSLEFT=$(( ($MEGS-$END)/$TOTALDOWNLOADED*$TIMEDIFF ))
|
||||
OVERALLDAYS=$(( $OVERALLSECONDSLEFT/60/60/24 ))
|
||||
OVERALLHOURS=$(( ($OVERALLSECONDSLEFT-$OVERALLDAYS*60*60*24)/60/60 ))
|
||||
OVERALLMINUTES=$(( ($OVERALLSECONDSLEFT-$OVERALLDAYS*60*60*24-$OVERALLHOURS*60*60)/60 ))
|
||||
OVERALLSECONDS=$(( $OVERALLSECONDSLEFT-$OVERALLDAYS*60*60*24-$OVERALLHOURS*60*60-$OVERALLMINUTES*60 ))
|
||||
echo -e "[${CURRENTTIME}] \033[0;32m${ENDDISPLAY}\e[0m of \033[0;32m${MEGSDISPLAY}\e[0m (\033[0;32m${CURRENTPCT}%\e[0m)\t ~${OVERALLMEGSPERSECOND}MB/s\t| ${OVERALLDAYS}d ${OVERALLHOURS}h ${OVERALLMINUTES}m ${OVERALLSECONDS}s remaining\t| Sleeping ${SLEEP}s..."
|
||||
TIMENOW=$(date +%s)
|
||||
TIMEDIFF=$(( TIMENOW - STARTTIME ))
|
||||
|
||||
DOWNLOADED=$(( END - START ))
|
||||
TOTALDOWNLOADED=$(( END - FIRST ))
|
||||
|
||||
CURRENTTIME=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
if (( DOWNLOADED <= 0 )); then
|
||||
echo "[${CURRENTTIME}] ${ENDDISPLAY} of ${MEGSDISPLAY} (${CURRENTPCT}%) No change detected | Sleeping ${SLEEP}s..."
|
||||
continue
|
||||
fi
|
||||
|
||||
if (( TIMEDIFF <= 0 || TOTALDOWNLOADED <= 0 )); then
|
||||
START="$END"
|
||||
continue
|
||||
fi
|
||||
|
||||
OVERALLMEGSPERSECOND=$(( TOTALDOWNLOADED / TIMEDIFF ))
|
||||
|
||||
if (( OVERALLMEGSPERSECOND < 1 )); then
|
||||
OVERALLMEGSPERSECOND=1
|
||||
fi
|
||||
|
||||
REMAINING=$(( MEGS - END ))
|
||||
|
||||
if (( REMAINING < 0 )); then
|
||||
REMAINING=0
|
||||
fi
|
||||
|
||||
OVERALLSECONDSLEFT=$(( REMAINING / OVERALLMEGSPERSECOND ))
|
||||
|
||||
OVERALLDAYS=$(( OVERALLSECONDSLEFT / 86400 ))
|
||||
OVERALLHOURS=$(( (OVERALLSECONDSLEFT % 86400) / 3600 ))
|
||||
OVERALLMINUTES=$(( (OVERALLSECONDSLEFT % 3600) / 60 ))
|
||||
OVERALLSECONDS=$(( OVERALLSECONDSLEFT % 60 ))
|
||||
|
||||
if (( $(echo "$MEGSGB > 0" | bc) )); then
|
||||
CURRENTPCT=$(echo "scale=0;($ENDGB/$MEGSGB)*100" | bc)
|
||||
else
|
||||
CURRENTPCT=0
|
||||
fi
|
||||
|
||||
echo "[${CURRENTTIME}] ${ENDDISPLAY} of ${MEGSDISPLAY} (${CURRENTPCT}%) ~${OVERALLMEGSPERSECOND}MB/s | ${OVERALLDAYS}d ${OVERALLHOURS}h ${OVERALLMINUTES}m ${OVERALLSECONDS}s remaining | Sleeping ${SLEEP}s..."
|
||||
|
||||
START="$END"
|
||||
|
||||
START=$END
|
||||
done
|
||||
Reference in new issue
Block a user