ackitsme
fac0d3202b
# 2.1.2 - Added EMAIL_TO and EMAIL_FROM variables for reports. # 2.1.3 - Added DOCKERUSER variable for account to use for launching contaners # 2.1.4 - Restic snapshot details (ID, parent, file/dir counts, data added, # total size, duration) are now written to the logs. # restic backup now runs with --json; snapshot size is read from the # backup summary instead of parsing 'restic snapshots' table output, # so it always refers to this run's snapshot. # Removed dependency on bc (awk used for arithmetic). # Added total new data uploaded to end of log.
511 lines
16 KiB
Bash
511 lines
16 KiB
Bash
#!/usr/bin/env bash
|
|
###############################################################################
|
|
### Docker Stack Backup Script
|
|
### Created by Charlie Silverman - ackitsme3@gmail.com
|
|
###############################################################################
|
|
### Version: 2.1.4
|
|
### Updated: 2026-10-26
|
|
### Purpose: Backs up all docker stacks in a given directory to Restic bucket
|
|
###############################################################################
|
|
### Requirements:
|
|
### Restic environment file: /root/.restic-env
|
|
### File format:
|
|
### export AWS_ACCESS_KEY_ID="[S3 Bucket Access Key ID]"
|
|
### export AWS_SECRET_ACCESS_KEY="[S3 Secret Access Key]"
|
|
### export RESTIC_REPOSITORY="Restic repository URL"
|
|
### export RESTIC_PASSWORD_FILE="Filepath to text file with restic password"
|
|
### Exception list: Optionally store a list of exceptions in the file
|
|
### dockerbackup-exception.list in the same directory
|
|
### as this file. File should be one entry per line.
|
|
### Each entry can contain wildcards if neeeded.
|
|
###############################################################################
|
|
# Version History:
|
|
# 1.0.0 - Initial build
|
|
# 1.1.0 - Added exception list via text file
|
|
# 1.1.1 - Added snapshot size to log
|
|
# 1.1.2 - Added aggregate backup size to end of log
|
|
# 1.1.3 - Added GiB to total backup size information
|
|
# 2.0.0 - Modified to save all files to a folder tree for backup rather than
|
|
# creating a single monolithic archive. This will allow restic to
|
|
# properly manage snapshots
|
|
# 2.0.1 - Changed exception lookup to case-insensitive
|
|
# 2.1.0 - Added email delivery of current execution log. Added
|
|
# success/failure/interruption notifications and run-specific logs.
|
|
# 2.1.1 - Added automatic cleanup of staging directories on backup failure
|
|
# or interruption.
|
|
# Added cleanup_staging() helper function.
|
|
# Added CURRENT_STAGING_DIR tracking.
|
|
# Added automatic RUN_LOG cleanup during failure and interruption.
|
|
# Improved housekeeping to prevent orphaned staging data after
|
|
# unexpected termination.
|
|
# 2.1.2 - Added EMAIL_TO and EMAIL_FROM variables for reports.
|
|
# 2.1.3 - Added DOCKERUSER variable for account to use for launching contaners
|
|
# 2.1.4 - Restic snapshot details (ID, parent, file/dir counts, data added,
|
|
# total size, duration) are now written to the logs.
|
|
# restic backup now runs with --json; snapshot size is read from the
|
|
# backup summary instead of parsing 'restic snapshots' table output,
|
|
# so it always refers to this run's snapshot.
|
|
# Removed dependency on bc (awk used for arithmetic).
|
|
# Added total new data uploaded to end of log.
|
|
|
|
set -Eeuo pipefail
|
|
|
|
###############################################################################
|
|
# Configuration
|
|
###############################################################################
|
|
|
|
COMPOSE_ROOT="/home/docker"
|
|
LOG_DIR="/home/docker/backuplogs"
|
|
mkdir -p "${LOG_DIR}"
|
|
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
|
|
EMAIL_TO="charlie@kiscc.com"
|
|
EMAIL_FROM="kisprodweb.dockerbackup@kiscc.com"
|
|
DOCKERUSER="csilverman"
|
|
|
|
# Restic repository configuration
|
|
export RESTIC_ENV_FILE="/root/.restic-env"
|
|
|
|
# Load Restic environment
|
|
source "$RESTIC_ENV_FILE"
|
|
|
|
HOSTNAME_SHORT="$(hostname -s)"
|
|
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
|
|
|
RUN_LOG="${LOG_DIR}/docker-backup_run_${TIMESTAMP}.log"
|
|
CURRENT_STAGING_DIR=""
|
|
RESTIC_OUTPUT=""
|
|
|
|
touch "${RUN_LOG}"
|
|
|
|
###############################################################################
|
|
# Verify Requirements
|
|
###############################################################################
|
|
|
|
for cmd in docker restic tar gzip jq awk; do
|
|
command -v "$cmd" >/dev/null 2>&1 || {
|
|
echo "Missing required command: $cmd"
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# Load exceptions, if any
|
|
if [ -f "${SCRIPT_DIR}/dockerbackup-exception.list" ]; then
|
|
mapfile -t SKIP_PROJECTS < "${SCRIPT_DIR}/dockerbackup-exception.list"
|
|
else
|
|
SKIP_PROJECTS=()
|
|
fi
|
|
|
|
|
|
###############################################################################
|
|
# Functions
|
|
###############################################################################
|
|
|
|
## Log to screen and log file
|
|
log() {
|
|
local message
|
|
message="[$(date '+%F %T')] $*"
|
|
|
|
echo "$message"
|
|
|
|
echo "$message" >> "$LOG_FILE"
|
|
echo "$message" >> "$RUN_LOG"
|
|
}
|
|
|
|
## Convert a byte count to a human-readable size (e.g. 1.23GiB)
|
|
human_bytes() {
|
|
local bytes="${1:-0}"
|
|
|
|
if command -v numfmt >/dev/null 2>&1; then
|
|
numfmt --to=iec-i --suffix=B --format="%.2f" "$bytes" 2>/dev/null \
|
|
|| echo "${bytes} B"
|
|
else
|
|
echo "${bytes} B"
|
|
fi
|
|
}
|
|
|
|
## Write Restic snapshot details from a backup summary (JSON) to the log
|
|
log_snapshot_info() {
|
|
local summary="$1"
|
|
|
|
local snapshot_id parent_id
|
|
local files_new files_changed files_unmodified
|
|
local dirs_new dirs_changed dirs_unmodified
|
|
local data_added data_added_packed
|
|
local total_files total_bytes total_duration
|
|
|
|
snapshot_id=$(jq -r '.snapshot_id // "none"' <<< "$summary")
|
|
parent_id=$(restic snapshots "$snapshot_id" --json 2>/dev/null \
|
|
| jq -r '.[0].parent // "none"' 2>/dev/null || echo "unknown")
|
|
|
|
files_new=$(jq -r '.files_new // 0' <<< "$summary")
|
|
files_changed=$(jq -r '.files_changed // 0' <<< "$summary")
|
|
files_unmodified=$(jq -r '.files_unmodified // 0' <<< "$summary")
|
|
dirs_new=$(jq -r '.dirs_new // 0' <<< "$summary")
|
|
dirs_changed=$(jq -r '.dirs_changed // 0' <<< "$summary")
|
|
dirs_unmodified=$(jq -r '.dirs_unmodified // 0' <<< "$summary")
|
|
data_added=$(jq -r '.data_added // 0' <<< "$summary")
|
|
data_added_packed=$(jq -r '.data_added_packed // empty' <<< "$summary")
|
|
total_files=$(jq -r '.total_files_processed // 0' <<< "$summary")
|
|
total_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
|
total_duration=$(jq -r '.total_duration // 0' <<< "$summary")
|
|
|
|
log "---------------- Restic Snapshot -----------------"
|
|
log "Snapshot ID: ${snapshot_id:0:8} (${snapshot_id})"
|
|
log "Parent snapshot: ${parent_id:0:8}"
|
|
log "Tags: docker, compose, ${project_name}, ${HOSTNAME_SHORT}"
|
|
log "Files: ${files_new} new, ${files_changed} changed, ${files_unmodified} unmodified"
|
|
log "Directories: ${dirs_new} new, ${dirs_changed} changed, ${dirs_unmodified} unmodified"
|
|
if [[ -n "$data_added_packed" ]]; then
|
|
log "Data added: $(human_bytes "$data_added") ($(human_bytes "$data_added_packed") stored)"
|
|
else
|
|
log "Data added: $(human_bytes "$data_added")"
|
|
fi
|
|
log "Snapshot size: $(human_bytes "$total_bytes") in ${total_files} files"
|
|
log "Duration: $(awk -v d="$total_duration" 'BEGIN{printf "%.1f", d}') seconds"
|
|
log "--------------------------------------------------"
|
|
}
|
|
|
|
## Check if specified project is in the exception list
|
|
should_skip() {
|
|
local project="$1"
|
|
|
|
shopt -s nocasematch
|
|
|
|
for pattern in "${SKIP_PROJECTS[@]}"; do
|
|
if [[ "$project" == $pattern ]]; then
|
|
shopt -u nocasematch
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
shopt -u nocasematch
|
|
return 1
|
|
}
|
|
|
|
## Restart composer stack
|
|
restart_stack() {
|
|
local compose_dir="$1"
|
|
|
|
log "Restarting stack: ${compose_dir}"
|
|
|
|
su - $DOCKERUSER -c \
|
|
"cd '${compose_dir}' && docker compose up -d" \
|
|
|| log "WARNING: Failed to restart stack ${compose_dir}"
|
|
}
|
|
|
|
## Send E-mail Notification
|
|
send_email_log() {
|
|
local subject="$1"
|
|
|
|
mailx -s "$subject" -r "$EMAIL_FROM" "$EMAIL_TO" < "$RUN_LOG"
|
|
}
|
|
|
|
## Cleanup Function
|
|
cleanup_staging() {
|
|
|
|
if [[ -n "${CURRENT_STAGING_DIR:-}" ]] && \
|
|
[[ -d "${CURRENT_STAGING_DIR}" ]]; then
|
|
|
|
log "Cleaning up staging directory: ${CURRENT_STAGING_DIR}"
|
|
|
|
rm -rf "${CURRENT_STAGING_DIR}" || \
|
|
log "WARNING: Failed to remove ${CURRENT_STAGING_DIR}"
|
|
|
|
fi
|
|
|
|
if [[ -n "${RESTIC_OUTPUT:-}" ]]; then
|
|
rm -f "${RESTIC_OUTPUT}"
|
|
fi
|
|
}
|
|
|
|
## Error Handler
|
|
error_handler() {
|
|
local exit_code=$?
|
|
|
|
trap - ERR
|
|
|
|
local line_number=$1
|
|
|
|
log "ERROR: Backup failed on line ${line_number} (exit code ${exit_code})"
|
|
|
|
cleanup_staging
|
|
|
|
send_email_log \
|
|
"FAILED: Docker Backup (${HOSTNAME_SHORT})"
|
|
|
|
rm -f "${RUN_LOG}"
|
|
|
|
exit "${exit_code}"
|
|
}
|
|
|
|
## Interrupt Handler
|
|
interrupt_handler() {
|
|
trap - SIGINT SIGTERM
|
|
|
|
log "ERROR: Backup interrupted"
|
|
|
|
cleanup_staging
|
|
|
|
send_email_log \
|
|
"INTERRUPTED: Docker Backup (${HOSTNAME_SHORT})"
|
|
|
|
rm -f "${RUN_LOG}"
|
|
|
|
exit 1
|
|
}
|
|
|
|
###############################################################################
|
|
# Trap Handlers
|
|
###############################################################################
|
|
trap 'error_handler $LINENO' ERR
|
|
trap interrupt_handler SIGINT SIGTERM
|
|
|
|
log "===================================================="
|
|
log "Docker Backup Starting"
|
|
log "Host: ${HOSTNAME_SHORT}"
|
|
log "Historical Log: ${LOG_FILE}"
|
|
log "Run Log: ${RUN_LOG}"
|
|
log "===================================================="
|
|
|
|
###############################################################################
|
|
# Locate Compose Files
|
|
###############################################################################
|
|
|
|
mapfile -t COMPOSE_FILES < <(
|
|
find "$COMPOSE_ROOT" \
|
|
\( -name "docker-compose.yml" \
|
|
-o -name "docker-compose.yaml" \
|
|
-o -name "compose.yml" \
|
|
-o -name "compose.yaml" \) \
|
|
| sort
|
|
)
|
|
|
|
if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then
|
|
log "No compose files found."
|
|
|
|
send_email_log \
|
|
"Docker Backup - No Compose Files (${HOSTNAME_SHORT})"
|
|
|
|
rm -f "${RUN_LOG}"
|
|
exit 0
|
|
fi
|
|
|
|
###############################################################################
|
|
# Process Each Compose Project
|
|
###############################################################################
|
|
|
|
total_size=0.00
|
|
total_added=0
|
|
|
|
for compose_file in "${COMPOSE_FILES[@]}"; do
|
|
|
|
compose_dir="$(dirname "$compose_file")"
|
|
project_name="$(basename "$compose_dir")"
|
|
|
|
if should_skip "$project_name"; then
|
|
log "Skipping project: ${project_name}"
|
|
continue
|
|
fi
|
|
|
|
log "=================================================="
|
|
log "Processing project: ${project_name}"
|
|
log "Compose file: ${compose_file}"
|
|
log "=================================================="
|
|
|
|
STAGING_DIR="/home/Backups/docker-backup-${project_name}"
|
|
CURRENT_STAGING_DIR="${STAGING_DIR}"
|
|
|
|
mkdir -p \
|
|
"${STAGING_DIR}/images" \
|
|
"${STAGING_DIR}/volumes" \
|
|
"${STAGING_DIR}/metadata"
|
|
|
|
###################################################
|
|
# Gather Information Before Shutdown
|
|
###################################################
|
|
|
|
cd "$compose_dir"
|
|
|
|
docker compose config --images \
|
|
| sort -u \
|
|
> "${STAGING_DIR}/metadata/images.list" || true
|
|
|
|
docker compose ps -q \
|
|
> "${STAGING_DIR}/metadata/containers.list" || true
|
|
|
|
###################################################
|
|
# Determine Volumes Used By Containers
|
|
###################################################
|
|
|
|
> "${STAGING_DIR}/metadata/volumes.list"
|
|
|
|
while read -r container_id; do
|
|
|
|
[ -z "$container_id" ] && continue
|
|
|
|
docker inspect "$container_id" \
|
|
--format '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{println}}{{end}}{{end}}'
|
|
|
|
done < "${STAGING_DIR}/metadata/containers.list" \
|
|
| sort -u \
|
|
> "${STAGING_DIR}/metadata/volumes.list"
|
|
|
|
###################################################
|
|
# Stop Stack
|
|
###################################################
|
|
|
|
log "Stopping stack..."
|
|
|
|
docker compose down --timeout 60
|
|
|
|
###################################################
|
|
# Backup Compose Files
|
|
###################################################
|
|
|
|
log "Backing up compose files..."
|
|
mkdir -p "${STAGING_DIR}/compose"
|
|
|
|
cp -a \
|
|
"${compose_dir}" \
|
|
"${STAGING_DIR}/compose/"
|
|
|
|
###################################################
|
|
# Backup Images
|
|
###################################################
|
|
|
|
log "Backing up images..."
|
|
|
|
while read -r image; do
|
|
[ -z "$image" ] && continue
|
|
|
|
if docker image inspect "$image" >/dev/null 2>&1; then
|
|
safe_name=$(echo "$image" | sed 's#[/:]#_#g')
|
|
|
|
log "Saving image: $image"
|
|
|
|
docker save "$image" \
|
|
> "${STAGING_DIR}/images/${safe_name}.tar"
|
|
fi
|
|
|
|
done < "${STAGING_DIR}/metadata/images.list"
|
|
|
|
###################################################
|
|
# Backup Volumes
|
|
###################################################
|
|
|
|
log "Backing up volumes..."
|
|
|
|
while read -r volume; do
|
|
[ -z "$volume" ] && continue
|
|
|
|
log "Saving volume: $volume"
|
|
|
|
docker run --rm \
|
|
-v "${volume}:/source:ro" \
|
|
-v "${STAGING_DIR}/volumes:/backup" \
|
|
alpine \
|
|
sh -c "cd /source && tar cf - . | tar xf - -C /backup"
|
|
|
|
done < "${STAGING_DIR}/metadata/volumes.list"
|
|
|
|
###################################################
|
|
# Metadata
|
|
###################################################
|
|
|
|
docker image ls \
|
|
> "${STAGING_DIR}/metadata/docker-images.txt"
|
|
|
|
docker volume ls \
|
|
> "${STAGING_DIR}/metadata/docker-volumes.txt"
|
|
|
|
docker network ls \
|
|
> "${STAGING_DIR}/metadata/docker-networks.txt"
|
|
|
|
docker compose config \
|
|
> "${STAGING_DIR}/metadata/rendered-compose.yml" || true
|
|
|
|
###################################################
|
|
# Restart Stack
|
|
###################################################
|
|
|
|
restart_stack "${compose_dir}"
|
|
|
|
###################################################
|
|
# Restic Backup
|
|
###################################################
|
|
|
|
log "Uploading project backup to Restic..."
|
|
|
|
# Capture restic's JSON output (outside the staging dir so it is
|
|
# not part of the backup). Errors/warnings still go to stderr.
|
|
RESTIC_OUTPUT="$(mktemp /tmp/restic-backup-XXXXXX.json)"
|
|
|
|
restic backup \
|
|
"${STAGING_DIR}" \
|
|
--json \
|
|
--tag docker \
|
|
--tag compose \
|
|
--tag "${project_name}" \
|
|
--tag "${HOSTNAME_SHORT}" \
|
|
> "${RESTIC_OUTPUT}"
|
|
|
|
###################################################
|
|
# Log snapshot details and get backup size
|
|
###################################################
|
|
|
|
summary=$(jq -cR 'fromjson? | select(.message_type == "summary")' \
|
|
"${RESTIC_OUTPUT}" | tail -n 1)
|
|
|
|
rm -f "${RESTIC_OUTPUT}"
|
|
RESTIC_OUTPUT=""
|
|
|
|
if [[ -n "$summary" ]]; then
|
|
log_snapshot_info "$summary"
|
|
|
|
snap_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
|
|
added_bytes=$(jq -r '.data_added // 0' <<< "$summary")
|
|
else
|
|
log "WARNING: Restic returned no snapshot summary for ${project_name}"
|
|
snap_bytes=0
|
|
added_bytes=0
|
|
fi
|
|
|
|
snapnum=$(awk -v b="$snap_bytes" 'BEGIN{printf "%.2f", b / 1073741824}')
|
|
|
|
###################################################
|
|
# Cleanup Local Backup
|
|
###################################################
|
|
|
|
rm -rf "${STAGING_DIR}"
|
|
CURRENT_STAGING_DIR=""
|
|
|
|
log "Project complete: ${project_name} ${snapnum} GiB"
|
|
log ""
|
|
total_size=$(awk -v a="$total_size" -v b="$snapnum" 'BEGIN{printf "%.2f", a + b}')
|
|
total_added=$(awk -v a="$total_added" -v b="$added_bytes" 'BEGIN{printf "%.0f", a + b}')
|
|
done
|
|
|
|
###############################################################################
|
|
# Retention Policy
|
|
###############################################################################
|
|
|
|
log "Applying Restic retention policy..."
|
|
|
|
restic forget \
|
|
--keep-daily 7 \
|
|
--keep-weekly 4 \
|
|
--keep-monthly 12 \
|
|
--prune
|
|
|
|
log "All compose projects backed up successfully."
|
|
log "Total Backup Size: $total_size GiB"
|
|
log "Total New Data Uploaded: $(human_bytes "$total_added")"
|
|
|
|
send_email_log \
|
|
"SUCCESS: Docker Backup (${HOSTNAME_SHORT})" || \
|
|
log "WARNING: Unable to send success email."
|
|
|
|
rm -f "${RUN_LOG}"
|