diff --git a/dockerbackup.sh b/dockerbackup.sh index 0db609c..9c443f5 100644 --- a/dockerbackup.sh +++ b/dockerbackup.sh @@ -3,7 +3,7 @@ ### Docker Stack Backup Script ### Created by Charlie Silverman - ackitsme3@gmail.com ############################################################################### -### Version: 2.0.1 +### Version: 2.1.1 ### Updated: 2026-10-18 ### Purpose: Backs up all docker stacks in a given directory to Restic bucket ############################################################################### @@ -29,6 +29,15 @@ # creating a single monolithic archive. This will allow restic to # properly manage snapshots # 2.0.1 - Changed exception lookup to case-insensitive +# 2.1.0 - Added email delivery of current execution log. Added +# success/failure/interruption notifications and run-specific logs. +# 2.1.1 - Added automatic cleanup of staging directories on backup failure +# or interruption. +# Added cleanup_staging() helper function. +# Added CURRENT_STAGING_DIR tracking. +# Added automatic RUN_LOG cleanup during failure and interruption. +# Improved housekeeping to prevent orphaned staging data after +# unexpected termination. set -Eeuo pipefail @@ -38,21 +47,41 @@ set -Eeuo pipefail COMPOSE_ROOT="/home/ackitsme/docker" LOG_DIR="/home/ackitsme/docker/backuplogs" +mkdir -p "${LOG_DIR}" LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log" # Restic repository configuration -export RESTIC_REPOSITORY="s3:https://s3.us-west-004.backblazeb2.com/ackitsbucket/docker" export RESTIC_ENV_FILE="/root/.restic-env" # Load Restic environment -source $RESTIC_ENV_FILE +source "$RESTIC_ENV_FILE" HOSTNAME_SHORT="$(hostname -s)" TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +RUN_LOG="${LOG_DIR}/docker-backup_run_${TIMESTAMP}.log" +CURRENT_STAGING_DIR="" + +touch "${RUN_LOG}" + +############################################################################### +# Verify Requirements +############################################################################### + +for cmd in docker restic tar gzip jq; do + command -v "$cmd" >/dev/null 2>&1 || { + echo "Missing required command: $cmd" + exit 1 + } +done + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Load exceptions, if any -if [ -f "dockerbackup-exception.list" ]; then - mapfile -t SKIP_PROJECTS < dockerbackup-exception.list +if [ -f "${SCRIPT_DIR}/dockerbackup-exception.list" ]; then + mapfile -t SKIP_PROJECTS < "${SCRIPT_DIR}/dockerbackup-exception.list" +else + SKIP_PROJECTS=() fi @@ -62,10 +91,13 @@ fi ## Log to screen and log file log() { - local message="[$(date '+%F %T')] $*" + local message + message="[$(date '+%F %T')] $*" - echo "$message" - echo "$message" >> "$LOG_FILE" + echo "$message" + + echo "$message" >> "$LOG_FILE" + echo "$message" >> "$RUN_LOG" } ## Check if specified project is in the exception list @@ -76,46 +108,109 @@ should_skip() { for pattern in "${SKIP_PROJECTS[@]}"; do if [[ "$project" == $pattern ]]; then + shopt -u nocasematch return 0 fi done shopt -u nocasematch - return 1 } -## Check if docker container is running -is_container_running() { - local container_name="$1" - - docker inspect -f '{{.State.Running}}' "$container_name" 2>/dev/null | grep -q '^true$' -} - ## Restart composer stack restart_stack() { - local compose_dir="$1" log "Restarting stack: ${compose_dir}" - ( - sudo -u ackitsme docker compose up -d - su - ackitsme -c "cd ${compose_dir};docker compose up -d" + su - ackitsme -c \ + "cd '${compose_dir}' && docker compose up -d" \ + || log "WARNING: Failed to restart stack ${compose_dir}" +} - ) || log "WARNING: Failed to restart stack ${compose_dir}" +## Send E-mail Notification +send_email_log() { + local subject="$1" + + if command -v mailx >/dev/null 2>&1; then + + mailx -s "$subject" \ + ackitsme3@gmail.com < "$RUN_LOG" + + elif command -v mail >/dev/null 2>&1; then + + mail -s "$subject" \ + ackitsme3@gmail.com < "$RUN_LOG" + + else + + log "WARNING: mail/mailx command not found. Email not sent." + + fi +} + +## Cleanup Function +cleanup_staging() { + + if [[ -n "${CURRENT_STAGING_DIR:-}" ]] && \ + [[ -d "${CURRENT_STAGING_DIR}" ]]; then + + log "Cleaning up staging directory: ${CURRENT_STAGING_DIR}" + + rm -rf "${CURRENT_STAGING_DIR}" || \ + log "WARNING: Failed to remove ${CURRENT_STAGING_DIR}" + + fi +} + +## Error Handler +error_handler() { + local exit_code=$? + + trap - ERR + + local line_number=$1 + + log "ERROR: Backup failed on line ${line_number} (exit code ${exit_code})" + + cleanup_staging + + send_email_log \ + "FAILED: Docker Backup (${HOSTNAME_SHORT})" + + rm -f "${RUN_LOG}" + + exit "${exit_code}" +} + +## Interrupt Handler +interrupt_handler() { + trap - SIGINT SIGTERM + + log "ERROR: Backup interrupted" + + cleanup_staging + + send_email_log \ + "INTERRUPTED: Docker Backup (${HOSTNAME_SHORT})" + + rm -f "${RUN_LOG}" + + exit 1 } ############################################################################### -# Verify Requirements +# Trap Handlers ############################################################################### +trap 'error_handler $LINENO' ERR +trap interrupt_handler SIGINT SIGTERM -for cmd in docker restic tar gzip; do - command -v "$cmd" >/dev/null 2>&1 || { - echo "Missing required command: $cmd" - exit 1 - } -done +log "====================================================" +log "Docker Backup Starting" +log "Host: ${HOSTNAME_SHORT}" +log "Historical Log: ${LOG_FILE}" +log "Run Log: ${RUN_LOG}" +log "====================================================" ############################################################################### # Locate Compose Files @@ -132,6 +227,11 @@ mapfile -t COMPOSE_FILES < <( if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then log "No compose files found." + + send_email_log \ + "Docker Backup - No Compose Files (${HOSTNAME_SHORT})" + + rm -f "${RUN_LOG}" exit 0 fi @@ -139,7 +239,7 @@ fi # Process Each Compose Project ############################################################################### -total_size=0 +total_size=0.00 for compose_file in "${COMPOSE_FILES[@]}"; do @@ -157,6 +257,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do log "==================================================" STAGING_DIR="/home/Backups/docker-backup-${project_name}" + CURRENT_STAGING_DIR="${STAGING_DIR}" mkdir -p \ "${STAGING_DIR}/images" \ @@ -219,17 +320,15 @@ for compose_file in "${COMPOSE_FILES[@]}"; do log "Backing up images..." while read -r image; do - [ -z "$image" ] && continue if docker image inspect "$image" >/dev/null 2>&1; then - safe_name=$(echo "$image" | sed 's#[/:]#_#g') log "Saving image: $image" - docker save "$image" \ - > "${STAGING_DIR}/images/${safe_name}.tar" + docker save "$image" \ + > "${STAGING_DIR}/images/${safe_name}.tar" fi done < "${STAGING_DIR}/metadata/images.list" @@ -241,7 +340,6 @@ for compose_file in "${COMPOSE_FILES[@]}"; do log "Backing up volumes..." while read -r volume; do - [ -z "$volume" ] && continue log "Saving volume: $volume" @@ -292,9 +390,11 @@ for compose_file in "${COMPOSE_FILES[@]}"; do ################################################### # Get backup size ################################################### - lastsnap=`restic ls latest` - lastsnapid=$(echo "$lastsnap" | awk '{print $2}') - snapdata=`restic snapshots $lastsnapid` +# lastsnap=$(restic ls latest) +# lastsnapid=$(echo "$lastsnap" | awk '{print $2}') + lastsnapid=$(restic snapshots --json | jq -r '.[-1].short_id') +# lastsnapid=$(restic snapshots latest --compact | awk 'NR==3 {print $1}') + snapdata=$(restic snapshots "$lastsnapid") snapnum=$(echo $snapdata | awk '{print $14}') snapsu=$(echo $snapdata | awk '{print $15}') case "$snapsu" in @@ -312,6 +412,7 @@ for compose_file in "${COMPOSE_FILES[@]}"; do ################################################### rm -rf "${STAGING_DIR}" + CURRENT_STAGING_DIR="" log "Project complete: ${project_name} ${snapnum} GiB" log "" @@ -332,3 +433,9 @@ restic forget \ log "All compose projects backed up successfully." log "Total Backup Size: $total_size GiB" + +send_email_log \ + "SUCCESS: Docker Backup (${HOSTNAME_SHORT})" || \ + log "WARNING: Unable to send success email." + +rm -f "${RUN_LOG}"