#!/bin/bash # # Federated Backup / Restore Tool to B2 Backblaze . /federated/lib/functions.sh usage() { cat << EOF Backup / Restore for Backblaze Usage: $0 [options] service Arguments: service Service to backup, system, nfs, or all for all services service - nextcloud,listmonk,espocrm,etc system - /federated/{bin,certs,lib,logs} dirs nfs - /mnt/mountpoint dirs all - system dirs + every app installed Options -l List files in a backup -b Perform backup of a service -d Delete backups of a service -r passkey Restore a backup of a service If passkey isn't specified we will use the passkey from the .gpg backblaze file EOF exit 2; } createbucket() { if [ ! -f "/federated/bin/.b2init" ]; then B2_APPLICATION_KEY_ID="$B2_APPLICATION_KEY_ID" B2_APPLICATION_KEY="$B2_APPLICATION_KEY" /federated/bin/b2-linux create_bucket $UUID allPrivate echo "$UUID" > /federated/bin/.b2init fi } listbackup_system() { echo "* Listing backup system files for /federated/{bin,certs,lib,logs}.." echo "* Listing backup system files for /federated/bin.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/bin/ [ $? -ne 0 ] && failcheck "* Couldn't list backup files" echo "* Listing backup system files for /federated/certs.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/certs/ [ $? -ne 0 ] && failcheck "* Couldn't list backup files" echo "* Listing backup system files for /federated/lib.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ [ $? -ne 0 ] && failcheck "* Couldn't list backup files" echo "* Listing backup system files for /federated/logs.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/logs/ [ $? -ne 0 ] && failcheck "* Couldn't list backup files" } listbackup_allservices() { echo "* Listing backup files for all services.." for i in "${SERVICES[@]}"; do echo "* Listing backup files for /federated/apps/$i.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't list backup files" done } listbackup_service() { echo "* Listing backup files for /federated/apps/$SERVICE.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't list backup files" } backup_service() { echo "* Backing up /federated/apps/$SERVICE.." # Dump only users and roles for postgresql service if [ "$SERVICE" = "postgresql" ]; then docker exec postgresql /bin/bash -c "pg_dumpall --globals-only -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$SERVICE.sql.gz" PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_* # Dump mysql databases elif [ "$SERVICE" = "pdnsmysql" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} mysql user | gzip -9 > /federated/apps/pdnsmysql/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsmysql/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "pdns" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdns/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdns/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "pdnsadmin" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdnsadmin/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsadmin/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "castopod" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} castopod | gzip -9 > /federated/apps/castopod/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/castopod/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "wordpress" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} wordpress | gzip -9 > /federated/apps/wordpress/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/wordpress/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "bookstack" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} bookstack | gzip -9 > /federated/apps/bookstack/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/bookstack/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "espocrm" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} espocrm | gzip -9 > /federated/apps/espocrm/dump_$SERVICE.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/espocrm/dump_$SERVICE.sql.gz else # For all other services, If the postgresql database exists for that service then dump it docker exec postgresql psql -U $SERVICE -c '\q' &>/dev/null if [ $? -eq 0 ]; then docker exec postgresql /bin/bash -c "pg_dump $SERVICE -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$SERVICE.sql.gz" [ $? -ne 0 ] && failcheck "* Couldn't pg_dump files" mv /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_$SERVICE.sql.gz /federated/apps/$SERVICE/ fi PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" [ -f "/federated/apps/$SERVICE/dump_$SERVICE.sql.gz" ] && rm /federated/apps/$SERVICE/dump_$SERVICE.sql.gz fi } backup_system() { echo "* Backing up system file /etc/federated.." cp /etc/federated /federated/bin/.etc.federated echo "* Backing up system files in /federated/{bin,certs,lib,logs}.." echo "* Backing up system files in /federated/bin.." PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/bin b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/bin/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" echo "* Backing up system files in /federated/certs.." PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/certs b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/certs/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" echo "* Backing up system files in /federated/lib.." PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/lib b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" echo "* Backing up system files in /federated/logs.." PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/logs b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/logs/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" } backup_allservices() { echo "* Backing up all services.." for i in "${SERVICES[@]}"; do # If app isn't installed then skip [ ! -d "/federated/apps/$i" ] && echo "$i not installed, skipping." && continue # Backup service echo "** Backing up /federated/apps/$i.." if [ "$i" = "postgresql" ]; then docker exec postgresql /bin/bash -c "pg_dumpall --globals-only -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$i.sql.gz" PASSPHRASE=$GPG_PASSPHRASE duplicity --full-if-older-than 1M /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_* elif [ "$i" = "pdnsmysql" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} mysql user | gzip -9 > /federated/apps/pdnsmysql/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity --full-if-older-than 1M /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsmysql/dump_$i.sql.gz elif [ "$i" = "pdns" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdns/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity --full-if-older-than 1M /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdns/dump_$i.sql.gz elif [ "$i" = "pdnsadmin" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdnsadmin/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity --full-if-older-than 1M /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsadmin/dump_$i.sql.gz elif [ "$i" = "castopod" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} castopod | gzip -9 > /federated/apps/castopod/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/castopod/dump_$i.sql.gz elif [ "$i" = "wordpress" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} wordpress | gzip -9 > /federated/apps/wordpress/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/wordpress/dump_$i.sql.gz elif [ "$i" = "bookstack" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} bookstack | gzip -9 > /federated/apps/bookstack/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/bookstack/dump_$i.sql.gz elif [ "$i" = "espocrm" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} espocrm | gzip -9 > /federated/apps/espocrm/dump_$i.sql.gz PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/espocrm/dump_$i.sql.gz else docker exec postgresql psql -U $i -c '\q' &>/dev/null if [ $? -eq 0 ]; then docker exec postgresql /bin/bash -c "pg_dump $i -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$i.sql.gz" [ $? -ne 0 ] && failcheck "* Couldn't pg_dump files" mv /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_$i.sql.gz /federated/apps/$i/ fi PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ [ $? -ne 0 ] && failcheck "* Couldn't backup files" [ -f "/federated/apps/$i/dump_$i.sql.gz" ] && rm /federated/apps/$i/dump_$i.sql.gz fi done } backup_nfsmounts() { # Backup NFS mounts if they exist NFSMOUNT=$(lsblk | grep mnt | awk '{ print $7 }') if [ "$NFSMOUNT" = "" ]; then echo "* No NFS mounts to backup."; else echo "* NFS mount found: $NFSMOUNT" echo "* Backing up NFS.." PASSPHRASE=$GPG_PASSPHRASE duplicity $NFSMOUNT b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/$NFSMOUNT/ [ $? -ne 0 ] && failcheck "* Couldn't backup NFS mount" fi } listbackup_nfsmounts() { # List NFS mounts if they exist NFSMOUNT=$(lsblk | grep mnt | awk '{ print $7 }') if [ "$NFSMOUNT" = "" ]; then echo "* No NFS mounts to list."; else echo "* NFS mount found: $NFSMOUNT" echo "* Listing NFS.." PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/$NFSMOUNT/ [ $? -ne 0 ] && failcheck "* Couldn't list NFS mount" fi } delete_backup() { echo "* Deleting backup files for /federated/apps/$SERVICE.." mkdir -p /federated/tmp/empty cd /federated/tmp/empty && /federated/bin/b2-linux sync --allowEmptySource --delete . b2://fed-customers/$UUID/federated/apps/$SERVICE/ [ $? -ne 0 ] && failcheck "* Couldn't delete files" cd /federated && rm -rf /federated/tmp/empty } restorebackup_system() { echo "* Restoring system files to /federated.restore/{bin,certs,lib,logs}.." echo "* Restoring system files to /federated.restore/bin.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/bin/ /federated.restore/bin [ $? -ne 0 ] && failcheck "* Couldn't restore files" echo "* Restoring system files to /federated.restore/certs.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/certs/ /federated.restore/certs [ $? -ne 0 ] && failcheck "* Couldn't restore files" echo "* Restoring system files to /federated.restore/lib.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ /federated.restore/lib [ $? -ne 0 ] && failcheck "* Couldn't restore files" echo "* Restoring system files to /federated.restore/logs.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ /federated.restore/logs [ $? -ne 0 ] && failcheck "* Couldn't restore files" } restorebackup_service() { echo "* Restoring $SERVICE to /federated.restore/apps/$SERVICE.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ /federated.restore/apps/$SERVICE [ $? -ne 0 ] && failcheck "* Couldn't restore files" } restorebackup_allservices() { echo "* Restoring all services.." for i in "${SERVICES[@]}"; do echo "** Restoring $i to /federated.restore/apps/$i.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ /federated.restore/apps/$i [ $? -ne 0 ] && failcheck "* Couldn't restore files" done } restorebackup_nfsmounts() { # Restore NFS mounts if they exist NFSMOUNT=$(lsblk | grep mnt | awk '{ print $7 }') if [ "$NFSMOUNT" = "" ]; then echo "* No NFS mounts to restore."; else echo "* NFS mount found: $NFSMOUNT" echo "* Restoring NFS.." PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/$NFSMOUNT/ /federated.restore/$NFSMOUNT [ $? -ne 0 ] && failcheck "* Couldn't restore NFS mount" fi } check_freedisk() { # Check if there is enough free disk space (needs 15G minimum) FREEDISK=$(df / | awk 'NR==2{print int($4/1024/1024)}') if [ $FREEDISK -lt 15 ]; then send_alert_backups failcheck "Disk is below 15G, can't backup." fi } while getopts "lbdr" OPTION; do case $OPTION in l) LIST_BACKUP="true"; ;; b) RUN_BACKUP="true"; ;; d) RUN_DELETE="true"; ;; r) RUN_RESTORE="true"; ;; *) usage; exit 2; esac done [ $# -gt 3 ] || [ $# -lt 1 ] && usage SERVICE=$2 if [ $# -eq 3 ]; then PASSKEY=$2 SERVICE=$3 fi if [ -f "/federated/bin/.gpg.backblaze" ]; then GPG_PASSPHRASE=`cat /federated/bin/.gpg.backblaze` else failcheck "/federated/bin/.gpg.backblaze doesn't exist" fi # Source the federated file if [ -f "/etc/federated" ]; then . /etc/federated [ -z "$B2_APPLICATION_KEY_ID" ] && failcheck "/etc/federated doesn't include B2_APPLICATION_KEY_ID for backups" [ -z "$B2_APPLICATION_KEY" ] && failcheck "/etc/federated doesn't include B2_APPLICATION_KEY for backups" [ -z "$DOMAIN" ] && failcheck "/etc/federated doesn't include DOMAIN for backups" UUID=$(hostname) else failcheck "Could not find an /etc/federated file." fi # Grab the MySQL pass we need for dumping backups MYSQL_PASSWORD=`cat /federated/apps/pdnsmysql/.env | grep MYSQL_ROOT_PASSWORD | awk -F= '{ print $2 }'` # If -d is specified then run delete backup [ $RUN_DELETE ] && delete_backup # If -l is specified then run list backup if [ $LIST_BACKUP ]; then if [ "$SERVICE" = "all" ]; then listbackup_system listbackup_allservices listbackup_nfsmounts elif [ "$SERVICE" = "system" ]; then listbackup_system elif [ "$SERVICE" = "nfs" ]; then listbackup_nfsmounts else if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then listbackup_service else failcheck "$SERVICE is not a valid service" fi fi fi # If -b is specified then run backup if [ $RUN_BACKUP ]; then check_freedisk if [ "$SERVICE" = "all" ]; then backup_system backup_allservices backup_nfsmounts elif [ "$SERVICE" = "system" ]; then backup_system elif [ "$SERVICE" = "nfs" ]; then backup_nfsmounts else if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then backup_service else failcheck "$SERVICE is not a valid service" fi fi fi # If -r is specified then run restore if [ $RUN_RESTORE ]; then check_freedisk if [ "$PASSKEY" = "" ]; then GPG_PASSPHRASE=`cat /federated/bin/.gpg.backblaze` else GPG_PASSPHRASE="$PASSKEY" fi if [ "$SERVICE" = "all" ]; then restorebackup_system restorebackup_allservices restorebackup_nfsmounts elif [ "$SERVICE" = "system" ]; then restorebackup_system elif [ "$SERVICE" = "nfs" ]; then restorebackup_nfsmounts else if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then restorebackup_service else failcheck "$SERVICE is not a valid service" fi fi fi