#!/usr/bin/env bash
###############################################################################
### Docker Stack Backup Script
### Created by Charlie Silverman - ackitsme3@gmail.com
###############################################################################
### Version: 2.1.4
### Updated: 2026-10-26
### Purpose:  Backs up all docker stacks in a given directory to Restic bucket
###############################################################################
### Requirements:
### Restic environment file:  /root/.restic-env
### File format:
###     export AWS_ACCESS_KEY_ID="[S3 Bucket Access Key ID]"
###     export AWS_SECRET_ACCESS_KEY="[S3 Secret Access Key]"
###     export RESTIC_REPOSITORY="Restic repository URL"
###     export RESTIC_PASSWORD_FILE="Filepath to text file with restic password"
### Exception list:  Optionally store a list of exceptions in the file
###                  dockerbackup-exception.list in the same directory
###                  as this file.  File should be one entry per line.
###                  Each entry can contain wildcards if neeeded.
###############################################################################
# Version History:
# 1.0.0 - Initial build
# 1.1.0 - Added exception list via text file
# 1.1.1 - Added snapshot size to log
# 1.1.2 - Added aggregate backup size to end of log
# 1.1.3 - Added GiB to total backup size information
# 2.0.0 - Modified to save all files to a folder tree for backup rather than
#         creating a single monolithic archive.  This will allow restic to
#         properly manage snapshots
# 2.0.1 - Changed exception lookup to case-insensitive
# 2.1.0 - Added email delivery of current execution log. Added
#         success/failure/interruption notifications and run-specific logs.
# 2.1.1 - Added automatic cleanup of staging directories on backup failure
#         or interruption.
#         Added cleanup_staging() helper function.
#         Added CURRENT_STAGING_DIR tracking.
#         Added automatic RUN_LOG cleanup during failure and interruption.
#         Improved housekeeping to prevent orphaned staging data after
#         unexpected termination.
# 2.1.2 - Added EMAIL_TO and EMAIL_FROM variables for reports.
# 2.1.3 - Added DOCKERUSER variable for account to use for launching contaners
# 2.1.4 - Restic snapshot details (ID, parent, file/dir counts, data added,
#         total size, duration) are now written to the logs.
#         restic backup now runs with --json; snapshot size is read from the
#         backup summary instead of parsing 'restic snapshots' table output,
#         so it always refers to this run's snapshot.
#         Removed dependency on bc (awk used for arithmetic).
#         Added total new data uploaded to end of log.

set -Eeuo pipefail

###############################################################################
# Configuration
###############################################################################

COMPOSE_ROOT="/home/docker"
LOG_DIR="/home/docker/backuplogs"
mkdir -p "${LOG_DIR}"
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
EMAIL_TO="charlie@kiscc.com"
EMAIL_FROM="kisprodweb.dockerbackup@kiscc.com"
DOCKERUSER="csilverman"

# Restic repository configuration
export RESTIC_ENV_FILE="/root/.restic-env"

# Load Restic environment
source "$RESTIC_ENV_FILE"

HOSTNAME_SHORT="$(hostname -s)"
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"

RUN_LOG="${LOG_DIR}/docker-backup_run_${TIMESTAMP}.log"
CURRENT_STAGING_DIR=""
RESTIC_OUTPUT=""

touch "${RUN_LOG}"

###############################################################################
# Verify Requirements
###############################################################################

for cmd in docker restic tar gzip jq awk; do
    command -v "$cmd" >/dev/null 2>&1 || {
        echo "Missing required command: $cmd"
        exit 1
    }
done

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Load exceptions, if any
if [ -f "${SCRIPT_DIR}/dockerbackup-exception.list" ]; then
    mapfile -t SKIP_PROJECTS < "${SCRIPT_DIR}/dockerbackup-exception.list"
else
    SKIP_PROJECTS=()
fi


###############################################################################
# Functions
###############################################################################

## Log to screen and log file
log() {
        local message
        message="[$(date '+%F %T')] $*"

    echo "$message"

    echo "$message" >> "$LOG_FILE"
    echo "$message" >> "$RUN_LOG"
}

## Convert a byte count to a human-readable size (e.g. 1.23GiB)
human_bytes() {
    local bytes="${1:-0}"

    if command -v numfmt >/dev/null 2>&1; then
        numfmt --to=iec-i --suffix=B --format="%.2f" "$bytes" 2>/dev/null \
            || echo "${bytes} B"
    else
        echo "${bytes} B"
    fi
}

## Write Restic snapshot details from a backup summary (JSON) to the log
log_snapshot_info() {
    local summary="$1"

    local snapshot_id parent_id
    local files_new files_changed files_unmodified
    local dirs_new dirs_changed dirs_unmodified
    local data_added data_added_packed
    local total_files total_bytes total_duration

    snapshot_id=$(jq -r '.snapshot_id // "none"' <<< "$summary")
    parent_id=$(restic snapshots "$snapshot_id" --json 2>/dev/null \
        | jq -r '.[0].parent // "none"' 2>/dev/null || echo "unknown")

    files_new=$(jq -r '.files_new // 0' <<< "$summary")
    files_changed=$(jq -r '.files_changed // 0' <<< "$summary")
    files_unmodified=$(jq -r '.files_unmodified // 0' <<< "$summary")
    dirs_new=$(jq -r '.dirs_new // 0' <<< "$summary")
    dirs_changed=$(jq -r '.dirs_changed // 0' <<< "$summary")
    dirs_unmodified=$(jq -r '.dirs_unmodified // 0' <<< "$summary")
    data_added=$(jq -r '.data_added // 0' <<< "$summary")
    data_added_packed=$(jq -r '.data_added_packed // empty' <<< "$summary")
    total_files=$(jq -r '.total_files_processed // 0' <<< "$summary")
    total_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
    total_duration=$(jq -r '.total_duration // 0' <<< "$summary")

    log "---------------- Restic Snapshot -----------------"
    log "Snapshot ID:      ${snapshot_id:0:8} (${snapshot_id})"
    log "Parent snapshot:  ${parent_id:0:8}"
    log "Tags:             docker, compose, ${project_name}, ${HOSTNAME_SHORT}"
    log "Files:            ${files_new} new, ${files_changed} changed, ${files_unmodified} unmodified"
    log "Directories:      ${dirs_new} new, ${dirs_changed} changed, ${dirs_unmodified} unmodified"
    if [[ -n "$data_added_packed" ]]; then
        log "Data added:       $(human_bytes "$data_added") ($(human_bytes "$data_added_packed") stored)"
    else
        log "Data added:       $(human_bytes "$data_added")"
    fi
    log "Snapshot size:    $(human_bytes "$total_bytes") in ${total_files} files"
    log "Duration:         $(awk -v d="$total_duration" 'BEGIN{printf "%.1f", d}') seconds"
    log "--------------------------------------------------"
}

## Check if specified project is in the exception list
should_skip() {
    local project="$1"

    shopt -s nocasematch

    for pattern in "${SKIP_PROJECTS[@]}"; do
        if [[ "$project" == $pattern ]]; then
            shopt -u nocasematch
            return 0
        fi
    done

    shopt -u nocasematch
    return 1
}

## Restart composer stack
restart_stack() {
    local compose_dir="$1"

    log "Restarting stack: ${compose_dir}"

    su - $DOCKERUSER -c \
        "cd '${compose_dir}' && docker compose up -d" \
        || log "WARNING: Failed to restart stack ${compose_dir}"
}

## Send E-mail Notification
send_email_log() {
    local subject="$1"

    mailx -s "$subject" -r "$EMAIL_FROM" "$EMAIL_TO" < "$RUN_LOG"
}

## Cleanup Function
cleanup_staging() {

    if [[ -n "${CURRENT_STAGING_DIR:-}" ]] && \
       [[ -d "${CURRENT_STAGING_DIR}" ]]; then

        log "Cleaning up staging directory: ${CURRENT_STAGING_DIR}"

        rm -rf "${CURRENT_STAGING_DIR}" || \
            log "WARNING: Failed to remove ${CURRENT_STAGING_DIR}"

    fi

    if [[ -n "${RESTIC_OUTPUT:-}" ]]; then
        rm -f "${RESTIC_OUTPUT}"
    fi
}

## Error Handler
error_handler() {
    local exit_code=$?

    trap - ERR

    local line_number=$1

    log "ERROR: Backup failed on line ${line_number} (exit code ${exit_code})"

    cleanup_staging

    send_email_log \
        "FAILED: Docker Backup (${HOSTNAME_SHORT})"

    rm -f "${RUN_LOG}"

    exit "${exit_code}"
}

## Interrupt Handler
interrupt_handler() {
        trap - SIGINT SIGTERM

    log "ERROR: Backup interrupted"

    cleanup_staging

    send_email_log \
        "INTERRUPTED: Docker Backup (${HOSTNAME_SHORT})"

        rm -f "${RUN_LOG}"

    exit 1
}

###############################################################################
# Trap Handlers
###############################################################################
trap 'error_handler $LINENO' ERR
trap interrupt_handler SIGINT SIGTERM

log "===================================================="
log "Docker Backup Starting"
log "Host: ${HOSTNAME_SHORT}"
log "Historical Log: ${LOG_FILE}"
log "Run Log: ${RUN_LOG}"
log "===================================================="

###############################################################################
# Locate Compose Files
###############################################################################

mapfile -t COMPOSE_FILES < <(
    find "$COMPOSE_ROOT" \
        \( -name "docker-compose.yml" \
        -o -name "docker-compose.yaml" \
        -o -name "compose.yml" \
        -o -name "compose.yaml" \) \
        | sort
)

if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then
    log "No compose files found."

    send_email_log \
        "Docker Backup - No Compose Files (${HOSTNAME_SHORT})"

    rm -f "${RUN_LOG}"
    exit 0
fi

###############################################################################
# Process Each Compose Project
###############################################################################

total_size=0.00
total_added=0

for compose_file in "${COMPOSE_FILES[@]}"; do

    compose_dir="$(dirname "$compose_file")"
    project_name="$(basename "$compose_dir")"

    if should_skip "$project_name"; then
        log "Skipping project: ${project_name}"
        continue
    fi

    log "=================================================="
    log "Processing project: ${project_name}"
    log "Compose file: ${compose_file}"
    log "=================================================="

    STAGING_DIR="/home/Backups/docker-backup-${project_name}"
        CURRENT_STAGING_DIR="${STAGING_DIR}"

    mkdir -p \
        "${STAGING_DIR}/images" \
        "${STAGING_DIR}/volumes" \
        "${STAGING_DIR}/metadata"

    ###################################################
    # Gather Information Before Shutdown
    ###################################################

    cd "$compose_dir"

    docker compose config --images \
        | sort -u \
        > "${STAGING_DIR}/metadata/images.list" || true

    docker compose ps -q \
        > "${STAGING_DIR}/metadata/containers.list" || true

    ###################################################
    # Determine Volumes Used By Containers
    ###################################################

    > "${STAGING_DIR}/metadata/volumes.list"

    while read -r container_id; do

        [ -z "$container_id" ] && continue

        docker inspect "$container_id" \
            --format '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{println}}{{end}}{{end}}'

    done < "${STAGING_DIR}/metadata/containers.list" \
        | sort -u \
        > "${STAGING_DIR}/metadata/volumes.list"

    ###################################################
    # Stop Stack
    ###################################################

    log "Stopping stack..."

    docker compose down --timeout 60

    ###################################################
    # Backup Compose Files
    ###################################################

        log "Backing up compose files..."
        mkdir -p "${STAGING_DIR}/compose"

        cp -a \
                "${compose_dir}" \
                "${STAGING_DIR}/compose/"

    ###################################################
    # Backup Images
    ###################################################

    log "Backing up images..."

    while read -r image; do
        [ -z "$image" ] && continue

        if docker image inspect "$image" >/dev/null 2>&1; then
            safe_name=$(echo "$image" | sed 's#[/:]#_#g')

            log "Saving image: $image"

                        docker save "$image" \
                                > "${STAGING_DIR}/images/${safe_name}.tar"
        fi

    done < "${STAGING_DIR}/metadata/images.list"

    ###################################################
    # Backup Volumes
    ###################################################

    log "Backing up volumes..."

    while read -r volume; do
        [ -z "$volume" ] && continue

        log "Saving volume: $volume"

        docker run --rm \
            -v "${volume}:/source:ro" \
            -v "${STAGING_DIR}/volumes:/backup" \
            alpine \
            sh -c "cd /source && tar cf - . | tar xf - -C /backup"

    done < "${STAGING_DIR}/metadata/volumes.list"

    ###################################################
    # Metadata
    ###################################################

    docker image ls \
        > "${STAGING_DIR}/metadata/docker-images.txt"

    docker volume ls \
        > "${STAGING_DIR}/metadata/docker-volumes.txt"

    docker network ls \
        > "${STAGING_DIR}/metadata/docker-networks.txt"

    docker compose config \
        > "${STAGING_DIR}/metadata/rendered-compose.yml" || true

    ###################################################
    # Restart Stack
    ###################################################

    restart_stack "${compose_dir}"

    ###################################################
    # Restic Backup
    ###################################################

    log "Uploading project backup to Restic..."

    # Capture restic's JSON output (outside the staging dir so it is
    # not part of the backup). Errors/warnings still go to stderr.
    RESTIC_OUTPUT="$(mktemp /tmp/restic-backup-XXXXXX.json)"

    restic backup \
        "${STAGING_DIR}" \
        --json \
        --tag docker \
        --tag compose \
        --tag "${project_name}" \
        --tag "${HOSTNAME_SHORT}" \
        > "${RESTIC_OUTPUT}"

    ###################################################
    # Log snapshot details and get backup size
    ###################################################

    summary=$(jq -cR 'fromjson? | select(.message_type == "summary")' \
        "${RESTIC_OUTPUT}" | tail -n 1)

    rm -f "${RESTIC_OUTPUT}"
    RESTIC_OUTPUT=""

    if [[ -n "$summary" ]]; then
        log_snapshot_info "$summary"

        snap_bytes=$(jq -r '.total_bytes_processed // 0' <<< "$summary")
        added_bytes=$(jq -r '.data_added // 0' <<< "$summary")
    else
        log "WARNING: Restic returned no snapshot summary for ${project_name}"
        snap_bytes=0
        added_bytes=0
    fi

    snapnum=$(awk -v b="$snap_bytes" 'BEGIN{printf "%.2f", b / 1073741824}')

    ###################################################
    # Cleanup Local Backup
    ###################################################

    rm -rf "${STAGING_DIR}"
        CURRENT_STAGING_DIR=""

    log "Project complete: ${project_name} ${snapnum} GiB"
    log ""
    total_size=$(awk -v a="$total_size" -v b="$snapnum" 'BEGIN{printf "%.2f", a + b}')
    total_added=$(awk -v a="$total_added" -v b="$added_bytes" 'BEGIN{printf "%.0f", a + b}')
done

###############################################################################
# Retention Policy
###############################################################################

log "Applying Restic retention policy..."

restic forget \
    --keep-daily 7 \
    --keep-weekly 4 \
    --keep-monthly 12 \
    --prune

log "All compose projects backed up successfully."
log "Total Backup Size: $total_size GiB"
log "Total New Data Uploaded: $(human_bytes "$total_added")"

send_email_log \
    "SUCCESS: Docker Backup (${HOSTNAME_SHORT})" || \
    log "WARNING: Unable to send success email."

rm -f "${RUN_LOG}"