Inital commit
This commit is contained in:
ackitsme committed
1 parent
429f485060
commit
df95d2ba11
3 files changed
+554
No files matched your search
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
AppName=$1
|
||||
AppUser=$2
|
||||
AppPassword=$3
|
||||
DATE=`date "+%Y%m%d_%H%M%S"`
|
||||
ArchiveFile=$AppName-$DATE.tar.bz
|
||||
|
||||
## Dump database
|
||||
if [ ! -f /home/$AppUser/SQLBackup ]; then
|
||||
mkdir /home/$AppUser/SQLBackup
|
||||
fi
|
||||
mysqldump -u $AppUser --password=$AppPassword $AppName > /home/$AppUser/SQLBackup/$AppName.sqldump
|
||||
|
||||
## Create bzipped tar archive
|
||||
tar cjf /home/Backups/$ArchiveFile /home/$AppUser/
|
||||
|
||||
## Create script to upload backup file to webbuilder.ackcentuate.com
|
||||
echo sftp ackitsme@webbuilder.ackcentuate.com \<\<\< \$\'put /home/Backups/$ArchiveFile\' > /tmp/$AppName-upload.sh
|
||||
chmod +x /tmp/$AppName-upload.sh
|
||||
|
||||
## Execupte upload script
|
||||
source /tmp/$AppName-upload.sh
|
||||
|
||||
## Remove upload script
|
||||
rm /tmp/$AppName-upload.sh
|
||||
|
||||
## Clean up Applicaiton Backups over 14 days old
|
||||
find /home/Backups/$AppName* -mtime +14 -exec rm {} \;
|
||||
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
### Docker Stack Backup Script
|
||||
### Created by Charlie Silverman - ackitsme3@gmail.com
|
||||
###############################################################################
|
||||
### Version: 1.1.0
|
||||
### Updated: 2017-07-26
|
||||
### Purpose: Backs up all docker stacks in a given directory to Restic bucket
|
||||
###############################################################################
|
||||
### Requirements:
|
||||
### Restic environment file: /root/.restic-env
|
||||
### File format:
|
||||
### export AWS_ACCESS_KEY_ID="[S3 Bucket Access Key ID]"
|
||||
### export AWS_SECRET_ACCESS_KEY="[S3 Secret Access Key]"
|
||||
### export RESTIC_REPOSITORY="Restic repository URL"
|
||||
### export RESTIC_PASSWORD_FILE="Filepath to text file with restic password"
|
||||
### Exception list: Optionally store a list of exceptions in the file
|
||||
### dockerbackup-exception.list in the same directory
|
||||
### as this file. File should be one entry per line.
|
||||
### Each entry can contain wildcards if neeeded.
|
||||
###############################################################################
|
||||
# Version History:
|
||||
# 1.0.0 - Initial build
|
||||
# 1.1.0 - Added exception list via text file
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
###############################################################################
|
||||
# Configuration
|
||||
###############################################################################
|
||||
|
||||
COMPOSE_ROOT="/home/ackitsme/docker"
|
||||
LOG_DIR="/home/ackitsme/docker/backuplogs"
|
||||
LOG_FILE="${LOG_DIR}/docker-backup_$(date +%Y%m%d-%H%M%S).log"
|
||||
|
||||
# Restic repository configuration
|
||||
export RESTIC_REPOSITORY="s3:https://s3.us-west-004.backblazeb2.com/ackitsbucket/docker"
|
||||
export RESTIC_ENV_FILE="/root/.restic-env"
|
||||
|
||||
# Load Restic environment
|
||||
source $RESTIC_ENV_FILE
|
||||
|
||||
HOSTNAME_SHORT="$(hostname -s)"
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
|
||||
# Load exceptions, if any
|
||||
if [ -f "dockerbackup-exception.list" ]; then
|
||||
mapfile -t SKIP_PROJECTS < dockerbackup-exception.list
|
||||
fi
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Functions
|
||||
###############################################################################
|
||||
|
||||
## Log to screen and log file
|
||||
log() {
|
||||
local message="[$(date '+%F %T')] $*"
|
||||
|
||||
echo "$message"
|
||||
echo "$message" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
## Check if specified project is in the exception list
|
||||
should_skip() {
|
||||
local project="$1"
|
||||
|
||||
for pattern in "${SKIP_PROJECTS[@]}"; do
|
||||
if [[ "$project" == $pattern ]]; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
## Check if docker container is running
|
||||
is_container_running() {
|
||||
local container_name="$1"
|
||||
|
||||
docker inspect -f '{{.State.Running}}' "$container_name" 2>/dev/null | grep -q '^true$'
|
||||
}
|
||||
|
||||
## Restart composer stack
|
||||
restart_stack() {
|
||||
|
||||
local compose_dir="$1"
|
||||
|
||||
log "Restarting stack: ${compose_dir}"
|
||||
|
||||
(
|
||||
sudo -u ackitsme docker compose up -d
|
||||
su - ackitsme -c "cd ${compose_dir};docker compose up -d"
|
||||
|
||||
) || log "WARNING: Failed to restart stack ${compose_dir}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Verify Requirements
|
||||
###############################################################################
|
||||
|
||||
for cmd in docker restic tar gzip; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "Missing required command: $cmd"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
# Locate Compose Files
|
||||
###############################################################################
|
||||
|
||||
mapfile -t COMPOSE_FILES < <(
|
||||
find "$COMPOSE_ROOT" \
|
||||
\( -name "docker-compose.yml" \
|
||||
-o -name "docker-compose.yaml" \
|
||||
-o -name "compose.yml" \
|
||||
-o -name "compose.yaml" \) \
|
||||
| sort
|
||||
)
|
||||
|
||||
if [ "${#COMPOSE_FILES[@]}" -eq 0 ]; then
|
||||
log "No compose files found."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
###############################################################################
|
||||
# Process Each Compose Project
|
||||
###############################################################################
|
||||
|
||||
for compose_file in "${COMPOSE_FILES[@]}"; do
|
||||
|
||||
compose_dir="$(dirname "$compose_file")"
|
||||
project_name="$(basename "$compose_dir")"
|
||||
|
||||
if should_skip "$project_name"; then
|
||||
log "Skipping project: ${project_name}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "=================================================="
|
||||
log "Processing project: ${project_name}"
|
||||
log "Compose file: ${compose_file}"
|
||||
log "=================================================="
|
||||
|
||||
STAGING_DIR="/tmp/docker-backup-${project_name}-${TIMESTAMP}"
|
||||
|
||||
mkdir -p \
|
||||
"${STAGING_DIR}/images" \
|
||||
"${STAGING_DIR}/volumes" \
|
||||
"${STAGING_DIR}/metadata"
|
||||
|
||||
###################################################
|
||||
# Gather Information Before Shutdown
|
||||
###################################################
|
||||
|
||||
cd "$compose_dir"
|
||||
|
||||
docker compose config --images \
|
||||
| sort -u \
|
||||
> "${STAGING_DIR}/metadata/images.list" || true
|
||||
|
||||
docker compose ps -q \
|
||||
> "${STAGING_DIR}/metadata/containers.list" || true
|
||||
|
||||
###################################################
|
||||
# Determine Volumes Used By Containers
|
||||
###################################################
|
||||
|
||||
> "${STAGING_DIR}/metadata/volumes.list"
|
||||
|
||||
while read -r container_id; do
|
||||
|
||||
[ -z "$container_id" ] && continue
|
||||
|
||||
docker inspect "$container_id" \
|
||||
--format '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}}{{println}}{{end}}{{end}}'
|
||||
|
||||
done < "${STAGING_DIR}/metadata/containers.list" \
|
||||
| sort -u \
|
||||
> "${STAGING_DIR}/metadata/volumes.list"
|
||||
|
||||
###################################################
|
||||
# Stop Stack
|
||||
###################################################
|
||||
|
||||
log "Stopping stack..."
|
||||
|
||||
docker compose down --timeout 60
|
||||
|
||||
###################################################
|
||||
# Backup Compose Files
|
||||
###################################################
|
||||
|
||||
log "Backing up compose files..."
|
||||
|
||||
tar -czf \
|
||||
"${STAGING_DIR}/${project_name}-compose.tar.gz" \
|
||||
-C "$(dirname "$compose_dir")" \
|
||||
"$(basename "$compose_dir")"
|
||||
|
||||
###################################################
|
||||
# Backup Images
|
||||
###################################################
|
||||
|
||||
log "Backing up images..."
|
||||
|
||||
while read -r image; do
|
||||
|
||||
[ -z "$image" ] && continue
|
||||
|
||||
if docker image inspect "$image" >/dev/null 2>&1; then
|
||||
|
||||
safe_name=$(echo "$image" | sed 's#[/:]#_#g')
|
||||
|
||||
log "Saving image: $image"
|
||||
|
||||
docker save "$image" \
|
||||
| gzip \
|
||||
> "${STAGING_DIR}/images/${safe_name}.tar.gz"
|
||||
fi
|
||||
|
||||
done < "${STAGING_DIR}/metadata/images.list"
|
||||
|
||||
###################################################
|
||||
# Backup Volumes
|
||||
###################################################
|
||||
|
||||
log "Backing up volumes..."
|
||||
|
||||
while read -r volume; do
|
||||
|
||||
[ -z "$volume" ] && continue
|
||||
|
||||
log "Saving volume: $volume"
|
||||
|
||||
docker run --rm \
|
||||
-v "${volume}:/source:ro" \
|
||||
-v "${STAGING_DIR}/volumes:/backup" \
|
||||
alpine \
|
||||
tar -czf "/backup/${volume}.tar.gz" -C /source .
|
||||
|
||||
done < "${STAGING_DIR}/metadata/volumes.list"
|
||||
|
||||
###################################################
|
||||
# Metadata
|
||||
###################################################
|
||||
|
||||
docker image ls \
|
||||
> "${STAGING_DIR}/metadata/docker-images.txt"
|
||||
|
||||
docker volume ls \
|
||||
> "${STAGING_DIR}/metadata/docker-volumes.txt"
|
||||
|
||||
docker network ls \
|
||||
> "${STAGING_DIR}/metadata/docker-networks.txt"
|
||||
|
||||
docker compose config \
|
||||
> "${STAGING_DIR}/metadata/rendered-compose.yml" || true
|
||||
|
||||
###################################################
|
||||
# Restart Stack
|
||||
###################################################
|
||||
|
||||
restart_stack "${compose_dir}"
|
||||
|
||||
###################################################
|
||||
# Restic Backup
|
||||
###################################################
|
||||
|
||||
log "Uploading project backup to Restic..."
|
||||
|
||||
restic backup \
|
||||
"${STAGING_DIR}" \
|
||||
--tag docker \
|
||||
--tag compose \
|
||||
--tag "${project_name}" \
|
||||
--tag "${HOSTNAME_SHORT}"
|
||||
|
||||
###################################################
|
||||
# Cleanup Local Backup
|
||||
###################################################
|
||||
|
||||
rm -rf "${STAGING_DIR}"
|
||||
|
||||
log "Project complete: ${project_name}"
|
||||
log ""
|
||||
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
# Retention Policy
|
||||
###############################################################################
|
||||
|
||||
log "Applying Restic retention policy..."
|
||||
|
||||
restic forget \
|
||||
--keep-daily 7 \
|
||||
--keep-weekly 4 \
|
||||
--keep-monthly 12 \
|
||||
--prune
|
||||
|
||||
log "All compose projects backed up successfully."
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env bash
|
||||
SCRIPTNAME="$(basename "$0")"
|
||||
VERSION=3.5.0
|
||||
FILEDATE="2026-07-26"
|
||||
|
||||
echo -e "\033[0m==============================================="
|
||||
echo -e " $SCRIPTNAME - rsync progress monitor"
|
||||
echo -e " Written by Charlie Silverman"
|
||||
echo -e " Version: $VERSION"
|
||||
echo -e " Release Date: $FILEDATE"
|
||||
echo -e "==============================================="
|
||||
|
||||
SRCFOLDER=""
|
||||
DSTFOLDER=""
|
||||
SLEEP=60
|
||||
HELP=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
echo $1
|
||||
IFS="=" read -r command value <<< "$1"
|
||||
echo "Command: $command"
|
||||
echo "Value: $value"
|
||||
echo "Value length: ${#value}"
|
||||
case "$command" in
|
||||
--source)
|
||||
if [ "${#value})" -gt 0 ]; then
|
||||
SRCFOLDER="$value"
|
||||
shift
|
||||
else
|
||||
SRCFOLDER="$2"
|
||||
shift 2
|
||||
fi
|
||||
;;
|
||||
--destination)
|
||||
if [ "${#value})" -gt 0 ]; then
|
||||
DSTFOLDER="$value"
|
||||
shift
|
||||
else
|
||||
DSTFOLDER="$2"
|
||||
shift 2
|
||||
fi
|
||||
;;
|
||||
--sleep)
|
||||
if [ "${#value})" -gt 0 ]; then
|
||||
SLEEP="$value"
|
||||
shift
|
||||
else
|
||||
SLEEP="$2"
|
||||
shift 2
|
||||
fi
|
||||
;;
|
||||
--help)
|
||||
HELP=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unknown argument: $1"
|
||||
HELP=true
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
ERROR=0
|
||||
|
||||
if [[ "$HELP" != "true" ]]; then
|
||||
# Check if source folder exists
|
||||
if [[ ! -d "${SRCFOLDER}" ]]; then
|
||||
echo -e "\n\033[0;31mE\e[4mRROR: Source Folder $SRCFOLDER does not exist.\e[0m"
|
||||
ERROR=1
|
||||
fi
|
||||
# Check if destination folder exists
|
||||
if [[ ! -d "${DSTFOLDER}" ]]; then
|
||||
echo -e "\n\033[0;31mE\e[4mRROR: Destination Folder $DSTFOLDER does not exist.\e[0m"
|
||||
ERROR=1
|
||||
fi
|
||||
# Verify that a sleep value was supplied
|
||||
if [ -z "${SLEEP}" ]; then
|
||||
echo -e "\n\033[0;31mE\e[4mRROR: Sleep delay not specified.\e[0m"
|
||||
ERROR=1
|
||||
else
|
||||
# Verify that the supplied sleep value is actually a Number
|
||||
re='^[0-9]+$'
|
||||
if ! [[ $SLEEP =~ $re ]]; then
|
||||
echo -e "\n\033[0;31m\e[4mERROR: Sleep delay invalid.\e[0m"
|
||||
ERROR=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Help display
|
||||
if $HELP; then
|
||||
cat << EOF
|
||||
|
||||
SYNOPSIS
|
||||
./$SCRIPTNAME --source <SRCFOLDER> --destination <DSTFOLDER> --sleep <SECONDS> [OPTIONS]"
|
||||
|
||||
OPTIONS
|
||||
--source <SRCFOLDER>
|
||||
Full or relative path to the source rsync folder to monitor
|
||||
|
||||
--destination <DSTFOLDER>
|
||||
Full or relative path to the destination rsync folder to monitor
|
||||
|
||||
--sleep <SECONDS>
|
||||
The number of seconds to sleep between scanning passes.
|
||||
Default: 60
|
||||
|
||||
--help
|
||||
Display this help message and exit.
|
||||
|
||||
EXAMPLE
|
||||
./$SCRIPTNAME --source /home/olduser --destination /home/newuser --sleep 300"
|
||||
EOF
|
||||
fi
|
||||
|
||||
echo "Calculating size of $SRCFOLDER..."
|
||||
|
||||
# Getting the mount point that relates to the source path, stripping off
|
||||
# the trailing slash if found
|
||||
if [[ "${SRCFOLDER: -1}" = "/" ]]; then
|
||||
SRCMOUNT=$(df ${SRCFOLDER%?} | sed -n '2p' | awk '{print $6}')
|
||||
else
|
||||
SRCMOUNT=$(df $SRCFOLDER | sed -n '2p' | awk '{print $6}')
|
||||
fi
|
||||
|
||||
# Determining the size of the source data. If the source path
|
||||
# matches a mount point from df, assuming that mount point data
|
||||
# to be accurate.
|
||||
if [ $SRCMOUNT != $SRCFOLDER ]; then
|
||||
MEGS=$(du -sm $SRCFOLDER |awk '{print $1}')
|
||||
else
|
||||
MEGS=$(df $SRCFOLDER | sed -n '2p' | awk '{print $3}')
|
||||
MEGS=$(($MEGS / 1024))
|
||||
fi
|
||||
MEGSDISPLAY="${MEGS}MB"
|
||||
MEGSGB=$(echo "scale=2;$MEGS/1024" | bc)
|
||||
if [ $(echo "$MEGSGB > 0" | bc) -ne 0 ]; then
|
||||
NUMFMT=$(printf "%'.3f\n" ${MEGSGB})
|
||||
MEGSDISPLAY="${NUMFMT}GB"
|
||||
fi
|
||||
|
||||
echo "Monitoring $DSTFOLDER..."
|
||||
|
||||
# Getting the mount point that relates to the destination path, stripping off
|
||||
# the trailing slash if found
|
||||
if [[ "${$DSTFOLDER: -1}" = "/" ]]; then
|
||||
DSTMOUNT=$(df ${DSTFOLDER%?} | sed -n '2p' | awk '{print $6}')
|
||||
else
|
||||
DSTMOUNT=$(df $DSTFOLDER | sed -n '2p' | awk '{print $6}')
|
||||
fi
|
||||
|
||||
SECONDSLEFT=1
|
||||
STARTTIME=$(date +%s)
|
||||
# Determining the current size of the destination data. If the
|
||||
# destination path matches a mount point from df, assuming that
|
||||
# mount point data to be accurate.
|
||||
if [ $DSTMOUNT != $DSTFOLDER ]; then
|
||||
FIRST=$(du -sm $DSTFOLDER |awk '{print $1}')
|
||||
else
|
||||
FIRST=$(df $DSTFOLDER | sed -n '2p' | awk '{print $3}')
|
||||
FIRST=$(($FIRST / 1024))
|
||||
fi
|
||||
|
||||
FIRSTDISPLAY="${FIRST}MB"
|
||||
FIRSTGB=$(echo "scale=2;$FIRST/1024" | bc)
|
||||
if [ $(echo "$FIRSTGB > 0" | bc) -ne 0 ]; then
|
||||
NUMFMT=$(printf "%'.3f\n" ${FIRSTGB})
|
||||
FIRSTDISPLAY="${NUMFMT}GB"
|
||||
fi
|
||||
|
||||
CURRENTPCT=$(echo "scale=2;($FIRSTGB/$MEGSGB)*100" | bc | cut -d . -f 1)
|
||||
|
||||
START=$FIRST
|
||||
CMDTIME=0
|
||||
CURRENTTIME=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo -e "[${CURRENTTIME}] \033[0;32m${FIRSTDISPLAY}\e[0m of \033[0;32m${MEGSDISPLAY}\e[0m (\033[0;32m${CURRENTPCT}%\e[0m)\tStarting\t\t| Sleeping ${SLEEP}s..."
|
||||
while (( $SECONDSLEFT > 0 )); do
|
||||
sleep $SLEEP
|
||||
|
||||
# Determining the current size of the destination data. If the
|
||||
# destination path matches a mount point from df, assuming that
|
||||
# mount point data to be accurate.
|
||||
if [ $DSTMOUNT != $DSTFOLDER ]; then
|
||||
END=$(du -sm $DSTFOLDER |awk '{print $1}')
|
||||
else
|
||||
END=$(df $DSTFOLDER | sed -n '2p' | awk '{print $3}')
|
||||
END=$(($END / 1024))
|
||||
fi
|
||||
|
||||
TIMENOW=$(date +%s)
|
||||
TIMEDIFF=$(($TIMENOW-$STARTTIME))
|
||||
DOWNLOADED=$(( $END-$START ))
|
||||
TOTALDOWNLOADED=$(( $END-$FIRST ))
|
||||
if [[ $DOWNLOADED -lt 0 ]]; then
|
||||
DOWNLOADED=0
|
||||
fi
|
||||
|
||||
ENDDISPLAY="${END}MB"
|
||||
ENDGB=$(echo "scale=2;$END/1024" | bc)
|
||||
if [[ $(echo "$END > 0" | bc) -ne 0 ]]; then
|
||||
NUMFMT=$(printf "%'.3f\n" ${ENDGB})
|
||||
ENDDISPLAY="${NUMFMT}GB"
|
||||
fi
|
||||
|
||||
CURRENTTIME=$(date '+%Y-%m0%d %H:%M:%S')/
|
||||
CURRENTPCT=$(echo "scale=2;($ENDGB/$MEGSGB)*100" | bc | cut -d . -f 1)
|
||||
|
||||
if [[ "$DOWNLOADED" == "0" ]]; then
|
||||
echo -e "[${CURRENTTIME}] \033[0;32m${ENDDISPLAY}\e[0m of \033[0;32m${MEGSDISPLAY}\e[0m (\033[0;32m${CURRENTPCT%}\e[0m)\t| No change since last check | Sleeping ${THISSLEEP}s..."
|
||||
continue
|
||||
fi
|
||||
|
||||
OVERALLMEGSPERSECOND=$(( $TOTALDOWNLOADED/$TIMEDIFF ))
|
||||
OVERALLSECONDSLEFT=$(( ($MEGS-$END)/$TOTALDOWNLOADED*$TIMEDIFF ))
|
||||
OVERALLDAYS=$(( $OVERALLSECONDSLEFT/60/60/24 ))
|
||||
OVERALLHOURS=$(( ($OVERALLSECONDSLEFT-$OVERALLDAYS*60*60*24)/60/60 ))
|
||||
OVERALLMINUTES=$(( ($OVERALLSECONDSLEFT-$OVERALLDAYS*60*60*24-$OVERALLHOURS*60*60)/60 ))
|
||||
OVERALLSECONDS=$(( $OVERALLSECONDSLEFT-$OVERALLDAYS*60*60*24-$OVERALLHOURS*60*60-$OVERALLMINUTES*60 ))
|
||||
echo -e "[${CURRENTTIME}] \033[0;32m${ENDDISPLAY}\e[0m of \033[0;32m${MEGSDISPLAY}\e[0m (\033[0;32m${CURRENTPCT}%\e[0m)\t ~${OVERALLMEGSPERSECOND}MB/s\t| ${OVERALLDAYS}d ${OVERALLHOURS}h ${OVERALLMINUTES}m ${OVERALLSECONDS}s remaining\t| Sleeping ${SLEEP}s..."
|
||||
|
||||
START=$END
|
||||
done
|
||||
Reference in new issue
Block a user