From 522c76cfc2f6bf5d37c9656dc11f18cf0f048a76 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 23 Jan 2024 16:20:16 +0000 Subject: [PATCH] Added new backuptool, restorevm --- bin/backuptool | 315 +++++++++++++++++++++++++++++++++++++---------- bin/restorevm | 28 +++++ lib/functions.sh | 5 + 3 files changed, 281 insertions(+), 67 deletions(-) create mode 100755 bin/restorevm diff --git a/bin/backuptool b/bin/backuptool index 61ec41c..6d3bad4 100755 --- a/bin/backuptool +++ b/bin/backuptool @@ -5,15 +5,23 @@ usage() { cat << EOF -Backup to Backblaze +Backup / Restore for Backblaze Usage: $0 [options] service Arguments: - service Service to backup or 'all' for all services + service Service to backup, system, nfs, or all for all services + + service - nextcloud,listmonk,espocrm,etc + system - /federated/{bin,certs,lib,logs} dirs + nfs - /mnt/mountpoint dirs + all - system dirs + every app installed + Options -l List files in a backup -b Perform backup of a service -d Delete backups of a service - -r Restore a backup of a service + -r passkey Restore a backup of a service + If passkey isn't specified we will use the passkey from + the .gpg backblaze file EOF exit 2; } @@ -23,66 +31,117 @@ createbucket() { echo "$UUID" > /federated/bin/.b2init fi } -listbackup() { - if [ "$SERVICE" = "all" ]; then - echo "* Listing backup files in all services.." - for i in "${SERVICES[@]}"; do - echo "* Listing backup files in $i.." - PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ - done - else - echo "* Listing backup files in $SERVICE.." - PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ - fi +listbackup_system() { + echo "* Listing backup system files for /federated/{bin,certs,lib,logs}.." + echo "* Listing backup system files for /federated/bin.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/bin/ + [ $? -ne 0 ] && failcheck "* Couldn't list backup files" + + echo "* Listing backup system files for /federated/certs.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/certs/ + [ $? -ne 0 ] && failcheck "* Couldn't list backup files" + + echo "* Listing backup system files for /federated/lib.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ + [ $? -ne 0 ] && failcheck "* Couldn't list backup files" + + echo "* Listing backup system files for /federated/logs.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/logs/ + [ $? -ne 0 ] && failcheck "* Couldn't list backup files" } -backupservice() { - echo "* Backing up $SERVICE.." +listbackup_allservices() { + echo "* Listing backup files for all services.." + for i in "${SERVICES[@]}"; do + echo "* Listing backup files for /federated/apps/$i.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't list backup files" + done +} +listbackup_service() { + echo "* Listing backup files for /federated/apps/$SERVICE.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't list backup files" +} +backup_service() { + echo "* Backing up /federated/apps/$SERVICE.." # Dump only users and roles for postgresql service if [ "$SERVICE" = "postgresql" ]; then docker exec postgresql /bin/bash -c "pg_dumpall --globals-only -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$SERVICE.sql.gz" - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_* + # Dump mysql databases elif [ "$SERVICE" = "pdnsmysql" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} mysql user | gzip -9 > /federated/apps/pdnsmysql/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsmysql/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "pdns" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdns/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdns/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "pdnsadmin" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdnsadmin/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsadmin/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "castopod" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} castopod | gzip -9 > /federated/apps/castopod/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/castopod/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "wordpress" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} wordpress | gzip -9 > /federated/apps/wordpress/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/wordpress/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "bookstack" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} bookstack | gzip -9 > /federated/apps/bookstack/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/bookstack/dump_$SERVICE.sql.gz elif [ "$SERVICE" = "espocrm" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} espocrm | gzip -9 > /federated/apps/espocrm/dump_$SERVICE.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/espocrm/dump_$SERVICE.sql.gz else # For all other services, If the postgresql database exists for that service then dump it docker exec postgresql psql -U $SERVICE -c '\q' &>/dev/null if [ $? -eq 0 ]; then docker exec postgresql /bin/bash -c "pg_dump $SERVICE -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$SERVICE.sql.gz" + [ $? -ne 0 ] && failcheck "* Couldn't pg_dump files" mv /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_$SERVICE.sql.gz /federated/apps/$SERVICE/ fi - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$SERVICE b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" [ -f "/federated/apps/$SERVICE/dump_$SERVICE.sql.gz" ] && rm /federated/apps/$SERVICE/dump_$SERVICE.sql.gz fi } -backupserviceall() { +backup_system() { + echo "* Backing up system file /etc/federated.." + cp /etc/federated /federated/bin/.etc.federated + + echo "* Backing up system files in /federated/{bin,certs,lib,logs}.." + echo "* Backing up system files in /federated/bin.." + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/bin b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/bin/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" + + echo "* Backing up system files in /federated/certs.." + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/certs b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/certs/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" + + echo "* Backing up system files in /federated/lib.." + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/lib b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" + + echo "* Backing up system files in /federated/logs.." + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/logs b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/logs/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" +} +backup_allservices() { echo "* Backing up all services.." for i in "${SERVICES[@]}"; do @@ -90,66 +149,142 @@ backupserviceall() { [ ! -d "/federated/apps/$i" ] && echo "$i not installed, skipping." && continue # Backup service - echo "** Backing up $i.." + echo "** Backing up /federated/apps/$i.." if [ "$i" = "postgresql" ]; then docker exec postgresql /bin/bash -c "pg_dumpall --globals-only -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$i.sql.gz" - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_* elif [ "$i" = "pdnsmysql" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} mysql user | gzip -9 > /federated/apps/pdnsmysql/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsmysql/dump_$i.sql.gz elif [ "$i" = "pdns" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdns/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdns/dump_$i.sql.gz elif [ "$i" = "pdnsadmin" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} pdns | gzip -9 > /federated/apps/pdnsadmin/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/pdnsadmin/dump_$i.sql.gz elif [ "$i" = "castopod" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} castopod | gzip -9 > /federated/apps/castopod/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/castopod/dump_$i.sql.gz elif [ "$i" = "wordpress" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} wordpress | gzip -9 > /federated/apps/wordpress/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/wordpress/dump_$i.sql.gz elif [ "$i" = "bookstack" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} bookstack | gzip -9 > /federated/apps/bookstack/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/bookstack/dump_$i.sql.gz elif [ "$i" = "espocrm" ]; then docker exec pdnsmysql mysqldump -uroot -p${MYSQL_PASSWORD} espocrm | gzip -9 > /federated/apps/espocrm/dump_$i.sql.gz - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" rm /federated/apps/espocrm/dump_$i.sql.gz else docker exec postgresql psql -U $i -c '\q' &>/dev/null if [ $? -eq 0 ]; then docker exec postgresql /bin/bash -c "pg_dump $i -c -U postgres | gzip -9 > /docker-entrypoint-initdb.d/dump_$i.sql.gz" + [ $? -ne 0 ] && failcheck "* Couldn't pg_dump files" mv /federated/apps/postgresql/data/docker-entrypoint-initdb.d/dump_$i.sql.gz /federated/apps/$i/ fi - PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ + PASSPHRASE=$GPG_PASSPHRASE duplicity /federated/apps/$i b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ + [ $? -ne 0 ] && failcheck "* Couldn't backup files" [ -f "/federated/apps/$i/dump_$i.sql.gz" ] && rm /federated/apps/$i/dump_$i.sql.gz fi done } -deletebackup() { +backup_nfsmounts() { + # Backup NFS mounts if they exist + NFSMOUNT=$(lsblk | grep mnt | awk '{ print $7 }') + if [ "$NFSMOUNT" = "" ]; then + echo "* No NFS mounts to backup."; + else + echo "* NFS mount found: $NFSMOUNT" + echo "* Backing up NFS.." + PASSPHRASE=$GPG_PASSPHRASE duplicity $NFSMOUNT b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/$NFSMOUNT/ + [ $? -ne 0 ] && failcheck "* Couldn't backup NFS mount" + fi +} +listbackup_nfsmounts() { + # List NFS mounts if they exist + NFSMOUNT=$(lsblk | grep mnt | awk '{ print $7 }') + if [ "$NFSMOUNT" = "" ]; then + echo "* No NFS mounts to list."; + else + echo "* NFS mount found: $NFSMOUNT" + echo "* Listing NFS.." + PASSPHRASE=$GPG_PASSPHRASE duplicity list-current-files b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/$NFSMOUNT/ + [ $? -ne 0 ] && failcheck "* Couldn't list NFS mount" + fi +} +delete_backup() { + echo "* Deleting backup files for /federated/apps/$SERVICE.." mkdir -p /federated/tmp/empty - cd /federated/tmp/empty && /federated/bin/b2-linux sync --allowEmptySource --delete . b2://$UUID/$SERVICE/ + cd /federated/tmp/empty && /federated/bin/b2-linux sync --allowEmptySource --delete . b2://fed-customers/$UUID/federated/apps/$SERVICE/ + [ $? -ne 0 ] && failcheck "* Couldn't delete files" cd /federated && rm -rf /federated/tmp/empty } -restorebackup() { - echo "* Restoring $SERVICE.." - PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$SERVICE/ /federated/apps/$SERVICE +restorebackup_system() { + echo "* Restoring system files to /federated.restore/{bin,certs,lib,logs}.." + echo "* Restoring system files to /federated.restore/bin.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/bin/ /federated.restore/bin + [ $? -ne 0 ] && failcheck "* Couldn't restore files" + + echo "* Restoring system files to /federated.restore/certs.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/certs/ /federated.restore/certs + [ $? -ne 0 ] && failcheck "* Couldn't restore files" + + echo "* Restoring system files to /federated.restore/lib.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ /federated.restore/lib + [ $? -ne 0 ] && failcheck "* Couldn't restore files" + + echo "* Restoring system files to /federated.restore/logs.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/lib/ /federated.restore/logs + [ $? -ne 0 ] && failcheck "* Couldn't restore files" } -restorebackupall() { +restorebackup_service() { + echo "* Restoring $SERVICE to /federated.restore/apps/$SERVICE.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$SERVICE/ /federated.restore/apps/$SERVICE + [ $? -ne 0 ] && failcheck "* Couldn't restore files" +} +restorebackup_allservices() { echo "* Restoring all services.." for i in "${SERVICES[@]}"; do - echo "** Restoring $i.." - PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@$UUID/$i/ /federated/apps/$i + echo "** Restoring $i to /federated.restore/apps/$i.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/federated/apps/$i/ /federated.restore/apps/$i + [ $? -ne 0 ] && failcheck "* Couldn't restore files" done } +restorebackup_nfsmounts() { + # Restore NFS mounts if they exist + NFSMOUNT=$(lsblk | grep mnt | awk '{ print $7 }') + if [ "$NFSMOUNT" = "" ]; then + echo "* No NFS mounts to restore."; + else + echo "* NFS mount found: $NFSMOUNT" + echo "* Restoring NFS.." + PASSPHRASE=$GPG_PASSPHRASE duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$UUID/$NFSMOUNT/ /federated.restore/$NFSMOUNT + [ $? -ne 0 ] && failcheck "* Couldn't restore NFS mount" + fi +} +check_freedisk() { + # Check if there is enough free disk space (needs 15G minimum) + FREEDISK=$(df / | awk 'NR==2{print int($4/1024/1024)}') + if [ $FREEDISK -lt 15 ]; then + send_alert_backups + failcheck "Disk is below 15G, can't backup." + fi +} while getopts "lbdr" OPTION; do case $OPTION in l) @@ -170,52 +305,98 @@ while getopts "lbdr" OPTION; do esac done -if [ $# -ne 2 ]; then - usage; -fi +[ $# -gt 3 ] || [ $# -lt 1 ] && usage SERVICE=$2 +if [ $# -eq 3 ]; then + PASSKEY=$2 + SERVICE=$3 +fi + if [ -f "/federated/bin/.gpg.backblaze" ]; then GPG_PASSPHRASE=`cat /federated/bin/.gpg.backblaze` else - GPG_PASSPHRASE=$(create_password); - echo "$GPG_PASSPHRASE" > /federated/bin/.gpg.backblaze + failcheck "/federated/bin/.gpg.backblaze doesn't exist" fi +# Source the federated file if [ -f "/etc/federated" ]; then . /etc/federated [ -z "$B2_APPLICATION_KEY_ID" ] && failcheck "/etc/federated doesn't include B2_APPLICATION_KEY_ID for backups" [ -z "$B2_APPLICATION_KEY" ] && failcheck "/etc/federated doesn't include B2_APPLICATION_KEY for backups" [ -z "$DOMAIN" ] && failcheck "/etc/federated doesn't include DOMAIN for backups" - UUID=`cat /etc/federated | grep DOMAIN | awk -F\. '{ print $1 }' | awk -F\" '{ print $2 }'` + UUID=$(hostname) else failcheck "Could not find an /etc/federated file." fi -get_installedapps -[ -f "/federated/bin/.b2init" ] && UUID=`cat /federated/bin/.b2init` -[ $LIST_BACKUP ] && listbackup -[ $RUN_DELETE ] && deletebackup +# Grab the MySQL pass we need for dumping backups MYSQL_PASSWORD=`cat /federated/apps/pdnsmysql/.env | grep MYSQL_ROOT_PASSWORD | awk -F= '{ print $2 }'` -if [ $RUN_RESTORE ]; then - if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then - restorebackup - elif [ "$SERVICE" = "all" ]; then - restorebackupall +# If -d is specified then run delete backup +[ $RUN_DELETE ] && delete_backup + +# If -l is specified then run list backup +if [ $LIST_BACKUP ]; then + if [ "$SERVICE" = "all" ]; then + listbackup_system + listbackup_allservices + listbackup_nfsmounts + elif [ "$SERVICE" = "system" ]; then + listbackup_system + elif [ "$SERVICE" = "nfs" ]; then + listbackup_nfsmounts else - failcheck "$SERVICE is not a valid service" + if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then + listbackup_service + else + failcheck "$SERVICE is not a valid service" + fi fi fi +# If -b is specified then run backup if [ $RUN_BACKUP ]; then - if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then - createbucket - backupservice - elif [ "$SERVICE" = "all" ]; then - createbucket - backupserviceall + check_freedisk + if [ "$SERVICE" = "all" ]; then + backup_system + backup_allservices + backup_nfsmounts + elif [ "$SERVICE" = "system" ]; then + backup_system + elif [ "$SERVICE" = "nfs" ]; then + backup_nfsmounts else - failcheck "$SERVICE is not a valid service" + if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then + backup_service + else + failcheck "$SERVICE is not a valid service" + fi + fi +fi + +# If -r is specified then run restore +if [ $RUN_RESTORE ]; then + check_freedisk + if [ "$PASSKEY" = "" ]; then + GPG_PASSPHRASE=`cat /federated/bin/.gpg.backblaze` + else + GPG_PASSPHRASE="$PASSKEY" + fi + + if [ "$SERVICE" = "all" ]; then + restorebackup_system + restorebackup_allservices + restorebackup_nfsmounts + elif [ "$SERVICE" = "system" ]; then + restorebackup_system + elif [ "$SERVICE" = "nfs" ]; then + restorebackup_nfsmounts + else + if printf '%s\0' "${SERVICES[@]}" | grep -Fxqz -- "$SERVICE"; then + restorebackup_service + else + failcheck "$SERVICE is not a valid service" + fi fi fi diff --git a/bin/restorevm b/bin/restorevm new file mode 100755 index 0000000..0d22ea6 --- /dev/null +++ b/bin/restorevm @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Restores data from backblaze for a particular VM id + +. /etc/federated +. /federated/lib/functions.sh + +usage() { +cat << EOF +Restore from Backblaze a specific VM id +Usage: $0 passkey service vmid +Arguments: + passkey - the GPG passkey for the VMID + service - service to restore: listmonk,nextcloud,etc + vmid - the VM id: f12345a1 + +EOF +exit 2; +} + +[ $# -ne 3 ] && usage +PASSKEY=$1 +SERVICE=$2 +VMID=$3 + +echo "* Restoring $SERVICE to /federated.restore.$VMID/apps/$SERVICE.." +PASSPHRASE=$PASSKEY duplicity --force b2://$B2_APPLICATION_KEY_ID:$B2_APPLICATION_KEY@fed-customers/$VMID/federated/apps/$SERVICE/ /federated.restore.$VMID/apps/$SERVICE +[ $? -ne 0 ] && failcheck "* Couldn't restore files" diff --git a/lib/functions.sh b/lib/functions.sh index b362c86..7913175 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -30,6 +30,11 @@ spin() { done done } +send_alert_backups() { + EXTERNALIP=`dig @resolver4.opendns.com myip.opendns.com +short 2> /dev/null` + echo "Generated by /federated/bin/backuptool" > /federated/apps/mail/data/root/certs/mailfile + docker exec mail bash -c "mail -r admin@$DOMAIN -a \"Content-type: text/html\" -s \"backuptool failed to run on $EXTERNALIP - low disk\" $ALERTS_EMAIL < /root/certs/mailfile" +} add_cron() { cat > /etc/logrotate.d/federated <