Updated to version 2.1.1
- Added email delivery of current execution log. - Added success/failure/interruption notifications and run-specific logs. - Added automatic cleanup of staging directories on backup failure or interruption. - Added cleanup_staging() helper function. - Added CURRENT_STAGING_DIR tracking. - Added automatic RUN_LOG cleanup during failure and interruption. Improved housekeeping to prevent orphaned staging data after unexpected termination.
This commit is contained in:
1 parent
baedfe9eea
commit
899eb11a2d
1 file changed
+140
-33
+140
-33
@@ -3,7 +3,7 @@
|
|||||||
### Docker Stack Backup Script
|
### Docker Stack Backup Script
|
||||||
### Created by Charlie Silverman - ackitsme3@gmail.com
|
### Created by Charlie Silverman - ackitsme3@gmail.com
|
||||||
###############################################################################
|
###############################################################################
|
||||||
### Version: 2.0.1
|
### Version: 2.1.1
|
||||||
### Updated: 2026-10-18
|
### Updated: 2026-10-18
|
||||||
### Purpose: Backs up all docker stacks in a given directory to Restic bucket
|
### Purpose: Backs up all docker stacks in a given directory to Restic bucket
|
||||||
###############################################################################
|
###############################################################################
|
||||||
@@ -29,6 +29,15 @@
|
|||||||
# creating a single monolithic archive. This will allow restic to
|
# creating a single monolithic archive. This will allow restic to
|
||||||
# properly manage snapshots
|
# properly manage snapshots
|
||||||
# 2.0.1 - Changed exception lookup to case-insensitive
|
# 2.0.1 - Changed exception lookup to case-insensitive
|
||||||
|
# 2.1.0 - Added email delivery of current execution log. Added
|
||||||
|
# success/failure/interruption notifications and run-specific logs.
|
||||||
|
# 2.1.1 - Added automatic cleanup of staging directories on backup failure
|
||||||
|
# or interruption.
|
||||||
|
# Added cleanup_staging() helper function.
|
||||||
|
# Added CURRENT_STAGING_DIR tracking.
|
||||||
|
# Added automatic RUN_LOG cleanup during failure and interruption.
|
||||||
|
# Improved housekeeping to prevent orphaned staging data after
|
||||||
|
# unexpected termination.
|
||||||
|
|
||||||
set -Eeuo pipefail
|
set -Eeuo pipefail
|
||||||
|
|
||||||
@@ -38,21 +47,41 @@ set -Eeuo pipefail
|
|||||||
|
|
||||||
COMPOSE_ROOT="/home/ackitsme/docker"
|
COMPOSE_ROOT="/home/ackitsme/docker"
|
||||||
LOG_DIR="/home/ackitsme/docker/backuplogs"
|
LOG_DIR="/home/ackitsme/docker/backuplogs"
|
||||||
|
mkdir -p "${LOG_DIR}"
|
||||||
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
|
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
|
||||||
|
|
||||||
# Restic repository configuration
|
# Restic repository configuration
|
||||||
export RESTIC_REPOSITORY="s3:https://s3.us-west-004.backblazeb2.com/ackitsbucket/docker"
|
|
||||||
export RESTIC_ENV_FILE="/root/.restic-env"
|
export RESTIC_ENV_FILE="/root/.restic-env"
|
||||||
|
|
||||||
# Load Restic environment
|
# Load Restic environment
|
||||||
source $RESTIC_ENV_FILE
|
source "$RESTIC_ENV_FILE"
|
||||||
|
|
||||||
HOSTNAME_SHORT="$(hostname -s)"
|
HOSTNAME_SHORT="$(hostname -s)"
|
||||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||||
|
|
||||||
|
RUN_LOG="${LOG_DIR}/docker-backup_run_${TIMESTAMP}.log"
|
||||||
|
CURRENT_STAGING_DIR=""
|
||||||
|
|
||||||
|
touch "${RUN_LOG}"
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Verify Requirements
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
for cmd in docker restic tar gzip jq; do
|
||||||
|
command -v "$cmd" >/dev/null 2>&1 || {
|
||||||
|
echo "Missing required command: $cmd"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
done
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
# Load exceptions, if any
|
# Load exceptions, if any
|
||||||
if [ -f "dockerbackup-exception.list" ]; then
|
if [ -f "${SCRIPT_DIR}/dockerbackup-exception.list" ]; then
|
||||||
mapfile -t SKIP_PROJECTS < dockerbackup-exception.list
|
mapfile -t SKIP_PROJECTS < "${SCRIPT_DIR}/dockerbackup-exception.list"
|
||||||
|
else
|
||||||
|
SKIP_PROJECTS=()
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
@@ -62,10 +91,13 @@ fi
|
|||||||
|
|
||||||
## Log to screen and log file
|
## Log to screen and log file
|
||||||
log() {
|
log() {
|
||||||
local message="[$(date '+%F %T')] $*"
|
local message
|
||||||
|
message="[$(date '+%F %T')] $*"
|
||||||
|
|
||||||
echo "$message"
|
echo "$message"
|
||||||
|
|
||||||
echo "$message" >> "$LOG_FILE"
|
echo "$message" >> "$LOG_FILE"
|
||||||
|
echo "$message" >> "$RUN_LOG"
|
||||||
}
|
}
|
||||||
|
|
||||||
## Check if specified project is in the exception list
|
## Check if specified project is in the exception list
|
||||||
@@ -76,46 +108,109 @@ should_skip() {
|
|||||||
|
|
||||||
for pattern in "${SKIP_PROJECTS[@]}"; do
|
for pattern in "${SKIP_PROJECTS[@]}"; do
|
||||||
if [[ "$project" == $pattern ]]; then
|
if [[ "$project" == $pattern ]]; then
|
||||||
|
shopt -u nocasematch
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
shopt -u nocasematch
|
shopt -u nocasematch
|
||||||
|
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
## Check if docker container is running
|
|
||||||
is_container_running() {
|
|
||||||
local container_name="$1"
|
|
||||||
|
|
||||||
docker inspect -f '{{.State.Running}}' "$container_name" 2>/dev/null | grep -q '^true$'
|
|
||||||
}
|
|
||||||
|
|
||||||
## Restart composer stack
|
## Restart composer stack
|
||||||
restart_stack() {
|
restart_stack() {
|
||||||
|
|
||||||
local compose_dir="$1"
|
local compose_dir="$1"
|
||||||
|
|
||||||
log "Restarting stack: ${compose_dir}"
|
log "Restarting stack: ${compose_dir}"
|
||||||
|
|
||||||
(
|
su - ackitsme -c \
|
||||||
sudo -u ackitsme docker compose up -d
|
"cd '${compose_dir}' && docker compose up -d" \
|
||||||
su - ackitsme -c "cd ${compose_dir};docker compose up -d"
|
|| log "WARNING: Failed to restart stack ${compose_dir}"
|
||||||
|
}
|
||||||
|
|
||||||
) || log "WARNING: Failed to restart stack ${compose_dir}"
|
## Send E-mail Notification
|
||||||
|
send_email_log() {
|
||||||
|
local subject="$1"
|
||||||
|
|
||||||
|
if command -v mailx >/dev/null 2>&1; then
|
||||||
|
|
||||||
|
mailx -s "$subject" \
|
||||||
|
ackitsme3@gmail.com < "$RUN_LOG"
|
||||||
|
|
||||||
|
elif command -v mail >/dev/null 2>&1; then
|
||||||
|
|
||||||
|
mail -s "$subject" \
|
||||||
|
ackitsme3@gmail.com < "$RUN_LOG"
|
||||||
|
|
||||||
|
else
|
||||||
|
|
||||||
|
log "WARNING: mail/mailx command not found. Email not sent."
|
||||||
|
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
## Cleanup Function
|
||||||
|
cleanup_staging() {
|
||||||
|
|
||||||
|
if [[ -n "${CURRENT_STAGING_DIR:-}" ]] && \
|
||||||
|
[[ -d "${CURRENT_STAGING_DIR}" ]]; then
|
||||||
|
|
||||||
|
log "Cleaning up staging directory: ${CURRENT_STAGING_DIR}"
|
||||||
|
|
||||||
|
rm -rf "${CURRENT_STAGING_DIR}" || \
|
||||||
|
log "WARNING: Failed to remove ${CURRENT_STAGING_DIR}"
|
||||||
|
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
## Error Handler
|
||||||
|
error_handler() {
|
||||||
|
local exit_code=$?
|
||||||
|
|
||||||
|
trap - ERR
|
||||||
|
|
||||||
|
local line_number=$1
|
||||||
|
|
||||||
|
log "ERROR: Backup failed on line ${line_number} (exit code ${exit_code})"
|
||||||
|
|
||||||
|
cleanup_staging
|
||||||
|
|
||||||
|
send_email_log \
|
||||||
|
"FAILED: Docker Backup (${HOSTNAME_SHORT})"
|
||||||
|
|
||||||
|
rm -f "${RUN_LOG}"
|
||||||
|
|
||||||
|
exit "${exit_code}"
|
||||||
|
}
|
||||||
|
|
||||||
|
## Interrupt Handler
|
||||||
|
interrupt_handler() {
|
||||||
|
trap - SIGINT SIGTERM
|
||||||
|
|
||||||
|
log "ERROR: Backup interrupted"
|
||||||
|
|
||||||
|
cleanup_staging
|
||||||
|
|
||||||
|
send_email_log \
|
||||||
|
"INTERRUPTED: Docker Backup (${HOSTNAME_SHORT})"
|
||||||
|
|
||||||
|
rm -f "${RUN_LOG}"
|
||||||
|
|
||||||
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# Verify Requirements
|
# Trap Handlers
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
trap 'error_handler $LINENO' ERR
|
||||||
|
trap interrupt_handler SIGINT SIGTERM
|
||||||
|
|
||||||
for cmd in docker restic tar gzip; do
|
log "===================================================="
|
||||||
command -v "$cmd" >/dev/null 2>&1 || {
|
log "Docker Backup Starting"
|
||||||
echo "Missing required command: $cmd"
|
log "Host: ${HOSTNAME_SHORT}"
|
||||||
exit 1
|
log "Historical Log: ${LOG_FILE}"
|
||||||
}
|
log "Run Log: ${RUN_LOG}"
|
||||||
done
|
log "===================================================="
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
# Locate Compose Files
|
# Locate Compose Files
|
||||||
@@ -132,6 +227,11 @@ mapfile -t COMPOSE_FILES < <(
|
|||||||
|
|
||||||
if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then
|
if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then
|
||||||
log "No compose files found."
|
log "No compose files found."
|
||||||
|
|
||||||
|
send_email_log \
|
||||||
|
"Docker Backup - No Compose Files (${HOSTNAME_SHORT})"
|
||||||
|
|
||||||
|
rm -f "${RUN_LOG}"
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -139,7 +239,7 @@ fi
|
|||||||
# Process Each Compose Project
|
# Process Each Compose Project
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
|
||||||
total_size=0
|
total_size=0.00
|
||||||
|
|
||||||
for compose_file in "${COMPOSE_FILES[@]}"; do
|
for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||||
|
|
||||||
@@ -157,6 +257,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
|||||||
log "=================================================="
|
log "=================================================="
|
||||||
|
|
||||||
STAGING_DIR="/home/Backups/docker-backup-${project_name}"
|
STAGING_DIR="/home/Backups/docker-backup-${project_name}"
|
||||||
|
CURRENT_STAGING_DIR="${STAGING_DIR}"
|
||||||
|
|
||||||
mkdir -p \
|
mkdir -p \
|
||||||
"${STAGING_DIR}/images" \
|
"${STAGING_DIR}/images" \
|
||||||
@@ -219,11 +320,9 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
|||||||
log "Backing up images..."
|
log "Backing up images..."
|
||||||
|
|
||||||
while read -r image; do
|
while read -r image; do
|
||||||
|
|
||||||
[ -z "$image" ] && continue
|
[ -z "$image" ] && continue
|
||||||
|
|
||||||
if docker image inspect "$image" >/dev/null 2>&1; then
|
if docker image inspect "$image" >/dev/null 2>&1; then
|
||||||
|
|
||||||
safe_name=$(echo "$image" | sed 's#[/:]#_#g')
|
safe_name=$(echo "$image" | sed 's#[/:]#_#g')
|
||||||
|
|
||||||
log "Saving image: $image"
|
log "Saving image: $image"
|
||||||
@@ -241,7 +340,6 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
|||||||
log "Backing up volumes..."
|
log "Backing up volumes..."
|
||||||
|
|
||||||
while read -r volume; do
|
while read -r volume; do
|
||||||
|
|
||||||
[ -z "$volume" ] && continue
|
[ -z "$volume" ] && continue
|
||||||
|
|
||||||
log "Saving volume: $volume"
|
log "Saving volume: $volume"
|
||||||
@@ -292,9 +390,11 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
|||||||
###################################################
|
###################################################
|
||||||
# Get backup size
|
# Get backup size
|
||||||
###################################################
|
###################################################
|
||||||
lastsnap=`restic ls latest`
|
# lastsnap=$(restic ls latest)
|
||||||
lastsnapid=$(echo "$lastsnap" | awk '{print $2}')
|
# lastsnapid=$(echo "$lastsnap" | awk '{print $2}')
|
||||||
snapdata=`restic snapshots $lastsnapid`
|
lastsnapid=$(restic snapshots --json | jq -r '.[-1].short_id')
|
||||||
|
# lastsnapid=$(restic snapshots latest --compact | awk 'NR==3 {print $1}')
|
||||||
|
snapdata=$(restic snapshots "$lastsnapid")
|
||||||
snapnum=$(echo $snapdata | awk '{print $14}')
|
snapnum=$(echo $snapdata | awk '{print $14}')
|
||||||
snapsu=$(echo $snapdata | awk '{print $15}')
|
snapsu=$(echo $snapdata | awk '{print $15}')
|
||||||
case "$snapsu" in
|
case "$snapsu" in
|
||||||
@@ -312,6 +412,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do
|
|||||||
###################################################
|
###################################################
|
||||||
|
|
||||||
rm -rf "${STAGING_DIR}"
|
rm -rf "${STAGING_DIR}"
|
||||||
|
CURRENT_STAGING_DIR=""
|
||||||
|
|
||||||
log "Project complete: ${project_name} ${snapnum} GiB"
|
log "Project complete: ${project_name} ${snapnum} GiB"
|
||||||
log ""
|
log ""
|
||||||
@@ -332,3 +433,9 @@ restic forget \
|
|||||||
|
|
||||||
log "All compose projects backed up successfully."
|
log "All compose projects backed up successfully."
|
||||||
log "Total Backup Size: $total_size GiB"
|
log "Total Backup Size: $total_size GiB"
|
||||||
|
|
||||||
|
send_email_log \
|
||||||
|
"SUCCESS: Docker Backup (${HOSTNAME_SHORT})" || \
|
||||||
|
log "WARNING: Unable to send success email."
|
||||||
|
|
||||||
|
rm -f "${RUN_LOG}"
|
||||||
Reference in new issue
Block a user