Automatisierte 3-2-1-Backups auf Netcup einrichten: SCP-Snapshots, atomare DB-Dumps & verschlüsseltes Restic

# 1. qemu-guest-agent installieren und aktivieren, damit das SCP bei Online-Snapshots fsfreeze ausführen kann
sudo apt update && sudo apt install -y qemu-guest-agent
sudo systemctl enable --now qemu-guest-agent
sudo systemctl status qemu-guest-agent --no-pager

# 2. Dateisystem-Belegung prüfen und ungenutzte Blöcke per fstrim freigeben, um CoW-Snapshots kompakt zu halten
df -h /
sudo fstrim -av

Active: active (running)
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1       252G   41G  201G  17% /
/: 198.4 GiB (213031256064 bytes) trimmed on /dev/vda1

# 1. Restic installieren und auf das neueste offizielle Binary aktualisieren
sudo apt update && sudo apt install -y restic bzip2 postgresql-client
sudo restic self-update

# 2. Root-exklusives Konfigurations- und Dump-Verzeichnis anlegen
sudo mkdir -p /etc/restic /var/backups/db-dumps
sudo chmod 0700 /etc/restic /var/backups/db-dumps

# 3. Kryptografisch starkes 48-Byte-Repository-Passwort generieren
openssl rand -base64 48 | sudo tee /etc/restic/repo.pw > /dev/null
sudo chmod 0400 /etc/restic/repo.pw

# 4. Offsite-S3- (oder SFTP-) Zugangsdaten und Go-Speicherlimit konfigurieren
sudo tee /etc/restic/backup.env > /dev/null <<'EOF'
export RESTIC_REPOSITORY="s3:https://s3.eu-central-003.backblazeb2.com/netcup-prod-backups/rs1000-nue"
export RESTIC_PASSWORD_FILE="/etc/restic/repo.pw"
export AWS_ACCESS_KEY_ID="YOUR_S3_KEY_ID"
export AWS_SECRET_ACCESS_KEY="YOUR_S3_APPLICATION_KEY"
export RESTIC_COMPRESSION="auto"
export GOGC=20
EOF
sudo chmod 0400 /etc/restic/backup.env

# 5. Verschlüsseltes Offsite-Restic-Repository initialisieren
sudo bash -c 'source /etc/restic/backup.env && restic init'

created restic repository 9c4e18f2a0 at s3:https://s3.eu-central-003.backblazeb2.com/netcup-prod-backups/rs1000-nue

sudo tee /usr/local/sbin/netcup-restic-backup > /dev/null <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

# Überlappende Backup-Läufe per Kernel-File-Lock (flock) verhindern
LOCKFILE="/var/run/netcup-restic-backup.lock"
exec 9>"${LOCKFILE}"
if ! flock -n 9; then
  echo "Another backup process is currently running. Exiting."
  exit 1
fi

source /etc/restic/backup.env
DUMP_DIR="/var/backups/db-dumps"
mkdir -p "${DUMP_DIR}"
chmod 0700 "${DUMP_DIR}"

echo "==> [1/4] Dumping running Docker PostgreSQL & MariaDB databases atomically..."
# Alle laufenden Postgres-Container konsistent im Custom-Format sichern
for PG_CONTAINER in $(docker ps --filter "ancestor=postgres:16.4-alpine3.20" --format "{{.Names}}"); do
  echo "  -> Dumping PostgreSQL container: ${PG_CONTAINER}"
  PG_USER=$(docker inspect "${PG_CONTAINER}" --format '{{range .Config.Env}}{{println .}}{{end}}' | awk -F= '/^POSTGRES_USER=/ {print $2}')
  PG_DB=$(docker inspect "${PG_CONTAINER}" --format '{{range .Config.Env}}{{println .}}{{end}}' | awk -F= '/^POSTGRES_DB=/ {print $2}')
  docker exec "${PG_CONTAINER}" pg_dump -U "${PG_USER:-postgres}" -d "${PG_DB:-postgres}" -Fc --Z=1 \
    > "${DUMP_DIR}/${PG_CONTAINER}_${PG_DB}.pgdump.tmp"
  mv "${DUMP_DIR}/${PG_CONTAINER}_${PG_DB}.pgdump.tmp" "${DUMP_DIR}/${PG_CONTAINER}_${PG_DB}.pgdump"
done

echo "==> [2/4] Executing encrypted Restic backup to off-site repository..."
restic backup \
  /etc \
  /opt/stacks \
  /var/backups/db-dumps \
  /home \
  /root \
  --tag "automated-daily" \
  --exclude="/var/lib/docker/overlay2" \
  --exclude="**/node_modules" \
  --exclude="**/.cache" \
  --exclude="**/*.log" \
  --one-file-system

echo "==> [3/4] Applying GFS retention policy (7 daily, 4 weekly, 6 monthly)..."
restic forget \
  --tag "automated-daily" \
  --keep-daily 7 \
  --keep-weekly 4 \
  --keep-monthly 6 \
  --prune

echo "==> [4/4] Verifying repository structure and 5% random data pack subset..."
restic check --read-data-subset=5%

echo "==> Backup & verification completed successfully at $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
EOF

sudo chmod 0700 /usr/local/sbin/netcup-restic-backup

# /etc/systemd/system/restic-backup.service
[Unit]
Description=Encrypted Off-Site Restic Backup & DB Dump for Netcup Server
After=network-online.target docker.service
Wants=network-online.target

[Service]
Type=oneshot
User=root
Nice=19
IOSchedulingClass=idle
IOSchedulingPriority=7
ExecStart=/usr/local/sbin/netcup-restic-backup
StandardOutput=journal
StandardError=journal

sudo tee /etc/systemd/system/restic-backup.timer > /dev/null <<'EOF'
[Unit]
Description=Run Netcup Restic Backup Daily at 03:15 AM with Randomized Jitter

[Timer]
OnCalendar=*-*-* 03:15:00
RandomizedDelaySec=1800
Persistent=true

[Install]
WantedBy=timers.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now restic-backup.timer

# Sofortigen Testlauf starten und systemd-Journal-Ausgabe prüfen
sudo systemctl start restic-backup.service
sudo journalctl -u restic-backup.service -n 40 --no-pager
sudo systemctl list-timers restic-backup.timer

NEXT                        LEFT          LAST                        PASSED       UNIT                ACTIVATES
Thu 2026-11-05 03:28:14 CET 6h left       Wed 2026-11-04 20:45:10 CET 12s ago      restic-backup.timer restic-backup.service

# 1. Alle verschlüsselten Snapshots im entfernten Repository auflisten
sudo bash -c 'source /etc/restic/backup.env && restic snapshots'

# 2. /opt/stacks und /var/backups/db-dumps aus dem neuesten Snapshot nach /tmp/restore-drill extrahieren
sudo mkdir -p /tmp/restore-drill
sudo bash -c 'source /etc/restic/backup.env && restic restore latest --target /tmp/restore-drill --include /opt/stacks --include /var/backups/db-dumps'

# 3. Integrität des wiederhergestellten PostgreSQL-Custom-Dumps prüfen (Inhaltsverzeichnis)
pg_restore --list /tmp/restore-drill/var/backups/db-dumps/*.pgdump | head -n 20

# 4. Temporäres Restore-Drill-Verzeichnis wieder entfernen
sudo rm -rf /tmp/restore-drill

ID        Time                 Host         Tags             Paths
------------------------------------------------------------------------------------
a81f39c2  2026-11-04 20:45:02  netcup-rs1   automated-daily  /etc, /opt/stacks, /var/backups/db-dumps

; Archive created at 2026-11-04 20:45:00 UTC
;     dbname: prod_app
;     TOC Entries: 48