Updated to version 2.1.1
- Added email delivery of current execution log. - Added success/failure/interruption notifications and run-specific logs. - Added automatic cleanup of staging directories on backup failure or interruption. - Added cleanup_staging() helper function. - Added CURRENT_STAGING_DIR tracking. - Added automatic RUN_LOG cleanup during failure and interruption. Improved housekeeping to prevent orphaned staging data after unexpected termination.
This commit is contained in:
ackitsme committed
1 parent
baedfe9eea
commit
899eb11a2d
1 file changed
+141
-34
+141
-34
@@ -3,7 +3,7 @@
|
||||
### Docker Stack Backup Script
|
||||
### Created by Charlie Silverman - ackitsme3@gmail.com
|
||||
###############################################################################
|
||||
### Version: 2.0.1
|
||||
### Version: 2.1.1
|
||||
### Updated: 2026-10-18
|
||||
### Purpose: Backs up all docker stacks in a given directory to Restic bucket
|
||||
###############################################################################
|
||||
@@ -29,6 +29,15 @@
|
||||
# creating a single monolithic archive. This will allow restic to
|
||||
# properly manage snapshots
|
||||
# 2.0.1 - Changed exception lookup to case-insensitive
|
||||
# 2.1.0 - Added email delivery of current execution log. Added
|
||||
# success/failure/interruption notifications and run-specific logs.
|
||||
# 2.1.1 - Added automatic cleanup of staging directories on backup failure
|
||||
# or interruption.
|
||||
# Added cleanup_staging() helper function.
|
||||
# Added CURRENT_STAGING_DIR tracking.
|
||||
# Added automatic RUN_LOG cleanup during failure and interruption.
|
||||
# Improved housekeeping to prevent orphaned staging data after
|
||||
# unexpected termination.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
@@ -38,21 +47,41 @@ set -Eeuo pipefail
|
||||
|
||||
COMPOSE_ROOT="/home/ackitsme/docker"
|
||||
LOG_DIR="/home/ackitsme/docker/backuplogs"
|
||||
mkdir -p "${LOG_DIR}"
|
||||
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
|
||||
|
||||
# Restic repository configuration
|
||||
export RESTIC_REPOSITORY="s3:https://s3.us-west-004.backblazeb2.com/ackitsbucket/docker"
|
||||
export RESTIC_ENV_FILE="/root/.restic-env"
|
||||
|
||||
# Load Restic environment
|
||||
source $RESTIC_ENV_FILE
|
||||
source "$RESTIC_ENV_FILE"
|
||||
|
||||
HOSTNAME_SHORT="$(hostname -s)"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
|
||||
RUN_LOG="${LOG_DIR}/docker-backup_run_${TIMESTAMP}.log"
|
||||
CURRENT_STAGING_DIR=""
|
||||
|
||||
touch "${RUN_LOG}"
|
||||
|
||||
###############################################################################
|
||||
# Verify Requirements
|
||||
###############################################################################
|
||||
|
||||
for cmd in docker restic tar gzip jq; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "Missing required command: $cmd"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Load exceptions, if any
|
||||
if [ -f "dockerbackup-exception.list" ]; then
|
||||
mapfile -t SKIP_PROJECTS < dockerbackup-exception.list
|
||||
if [ -f "${SCRIPT_DIR}/dockerbackup-exception.list" ]; then
|
||||
mapfile -t SKIP_PROJECTS < "${SCRIPT_DIR}/dockerbackup-exception.list"
|
||||
else
|
||||
SKIP_PROJECTS=()
|
||||
fi
|
||||
|
||||
|
||||
@@ -62,10 +91,13 @@ fi
|
||||
|
||||
## Log to screen and log file
|
||||
log() {
|
||||
local message="[$(date '+%F %T')] $*"
|
||||
local message
|
||||
message="[$(date '+%F %T')] $*"
|
||||
|
||||
echo "$message"
|
||||
|
||||
echo "$message" >> "$LOG_FILE"
|
||||
echo "$message" >> "$RUN_LOG"
|
||||
}
|
||||
|
||||
## Check if specified project is in the exception list
|
||||
@@ -76,46 +108,109 @@ should_skip() {
|
||||
|
||||
for pattern in "${SKIP_PROJECTS[@]}"; do
|
||||
if [[ "$project" == $pattern ]]; then
|
||||
shopt -u nocasematch
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
shopt -u nocasematch
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
## Check if docker container is running
|
||||
is_container_running() {
|
||||
local container_name="$1"
|
||||
|
||||
docker inspect -f '{{.State.Running}}' "$container_name" 2>/dev/null | grep -q '^true$'
|
||||
}
|
||||
|
||||
## Restart composer stack
|
||||
restart_stack() {
|
||||
|
||||
local compose_dir="$1"
|
||||
|
||||
log "Restarting stack: ${compose_dir}"
|
||||
|
||||
(
|
||||
sudo -u ackitsme docker compose up -d
|
||||
su - ackitsme -c "cd ${compose_dir};docker compose up -d"
|
||||
|
||||
) || log "WARNING: Failed to restart stack ${compose_dir}"
|
||||
su - ackitsme -c \
|
||||
"cd '${compose_dir}' && docker compose up -d" \
|
||||
|| log "WARNING: Failed to restart stack ${compose_dir}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Verify Requirements
|
||||
###############################################################################
|
||||
## Send E-mail Notification
|
||||
send_email_log() {
|
||||
local subject="$1"
|
||||
|
||||
if command -v mailx >/dev/null 2>&1; then
|
||||
|
||||
mailx -s "$subject" \
|
||||
ackitsme3@gmail.com < "$RUN_LOG"
|
||||
|
||||
elif command -v mail >/dev/null 2>&1; then
|
||||
|
||||
mail -s "$subject" \
|
||||
ackitsme3@gmail.com < "$RUN_LOG"
|
||||
|
||||
else
|
||||
|
||||
log "WARNING: mail/mailx command not found. Email not sent."
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
## Cleanup Function
|
||||
cleanup_staging() {
|
||||
|
||||
if [[ -n "${CURRENT_STAGING_DIR:-}" ]] && \
|
||||
[[ -d "${CURRENT_STAGING_DIR}" ]]; then
|
||||
|
||||
log "Cleaning up staging directory: ${CURRENT_STAGING_DIR}"
|
||||
|
||||
rm -rf "${CURRENT_STAGING_DIR}" || \
|
||||
log "WARNING: Failed to remove ${CURRENT_STAGING_DIR}"
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
## Error Handler
|
||||
error_handler() {
|
||||
local exit_code=$?
|
||||
|
||||
trap - ERR
|
||||
|
||||
local line_number=$1
|
||||
|
||||
log "ERROR: Backup failed on line ${line_number} (exit code ${exit_code})"
|
||||
|
||||
cleanup_staging
|
||||
|
||||
send_email_log \
|
||||
"FAILED: Docker Backup (${HOSTNAME_SHORT})"
|
||||
|
||||
rm -f "${RUN_LOG}"
|
||||
|
||||
exit "${exit_code}"
|
||||
}
|
||||
|
||||
## Interrupt Handler
|
||||
interrupt_handler() {
|
||||
trap - SIGINT SIGTERM
|
||||
|
||||
log "ERROR: Backup interrupted"
|
||||
|
||||
cleanup_staging
|
||||
|
||||
send_email_log \
|
||||
"INTERRUPTED: Docker Backup (${HOSTNAME_SHORT})"
|
||||
|
||||
rm -f "${RUN_LOG}"
|
||||
|
||||
for cmd in docker restic tar gzip; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "Missing required command: $cmd"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
# Trap Handlers
|
||||
###############################################################################
|
||||
trap 'error_handler $LINENO' ERR
|
||||
trap interrupt_handler SIGINT SIGTERM
|
||||
|
||||
log "===================================================="
|
||||
log "Docker Backup Starting"
|
||||
log "Host: ${HOSTNAME_SHORT}"
|
||||
log "Historical Log: ${LOG_FILE}"
|
||||
log "Run Log: ${RUN_LOG}"
|
||||
log "===================================================="
|
||||
|
||||
###############################################################################
|
||||
# Locate Compose Files
|
||||
@@ -132,6 +227,11 @@ mapfile -t COMPOSE_FILES < <(
|
||||
|
||||
if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then
|
||||
log "No compose files found."
|
||||
|
||||
send_email_log \
|
||||
"Docker Backup - No Compose Files (${HOSTNAME_SHORT})"
|
||||
|
||||
rm -f "${RUN_LOG}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -139,7 +239,7 @@ fi
|
||||
# Process Each Compose Project
|
||||
###############################################################################
|
||||
|
||||
total_size=0
|
||||
total_size=0.00
|
||||
|
||||
for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
|
||||
@@ -157,6 +257,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
log "=================================================="
|
||||
|
||||
STAGING_DIR="/home/Backups/docker-backup-${project_name}"
|
||||
CURRENT_STAGING_DIR="${STAGING_DIR}"
|
||||
|
||||
mkdir -p \
|
||||
"${STAGING_DIR}/images" \
|
||||
@@ -219,11 +320,9 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
log "Backing up images..."
|
||||
|
||||
while read -r image; do
|
||||
|
||||
[ -z "$image" ] && continue
|
||||
|
||||
if docker image inspect "$image" >/dev/null 2>&1; then
|
||||
|
||||
safe_name=$(echo "$image" | sed 's#[/:]#_#g')
|
||||
|
||||
log "Saving image: $image"
|
||||
@@ -241,7 +340,6 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
log "Backing up volumes..."
|
||||
|
||||
while read -r volume; do
|
||||
|
||||
[ -z "$volume" ] && continue
|
||||
|
||||
log "Saving volume: $volume"
|
||||
@@ -292,9 +390,11 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
###################################################
|
||||
# Get backup size
|
||||
###################################################
|
||||
lastsnap=`restic ls latest`
|
||||
lastsnapid=$(echo "$lastsnap" | awk '{print $2}')
|
||||
snapdata=`restic snapshots $lastsnapid`
|
||||
# lastsnap=$(restic ls latest)
|
||||
# lastsnapid=$(echo "$lastsnap" | awk '{print $2}')
|
||||
lastsnapid=$(restic snapshots --json | jq -r '.[-1].short_id')
|
||||
# lastsnapid=$(restic snapshots latest --compact | awk 'NR==3 {print $1}')
|
||||
snapdata=$(restic snapshots "$lastsnapid")
|
||||
snapnum=$(echo $snapdata | awk '{print $14}')
|
||||
snapsu=$(echo $snapdata | awk '{print $15}')
|
||||
case "$snapsu" in
|
||||
@@ -312,6 +412,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
###################################################
|
||||
|
||||
rm -rf "${STAGING_DIR}"
|
||||
CURRENT_STAGING_DIR=""
|
||||
|
||||
log "Project complete: ${project_name} ${snapnum} GiB"
|
||||
log ""
|
||||
@@ -332,3 +433,9 @@ restic forget \
|
||||
|
||||
log "All compose projects backed up successfully."
|
||||
log "Total Backup Size: $total_size GiB"
|
||||
|
||||
send_email_log \
|
||||
"SUCCESS: Docker Backup (${HOSTNAME_SHORT})" || \
|
||||
log "WARNING: Unable to send success email."
|
||||
|
||||
rm -f "${RUN_LOG}"
|
||||
Reference in new issue
Block a user