如何在 Netcup 上构建 3-2-1 自动化备份体系：SCP 快照、数据库原子转储与 Restic 加密异地归档

# 1. 安装并核验 qemu-guest-agent，以便 SCP 在创建在线快照时自动调用 fsfreeze 冻结文件系统
sudo apt update && sudo apt install -y qemu-guest-agent
sudo systemctl enable --now qemu-guest-agent
sudo systemctl status qemu-guest-agent --no-pager

# 2. 检查根文件系统占用并通过 fstrim 回收未使用的空闲块，使 CoW 快照保持紧凑
df -h /
sudo fstrim -av

Active: active (running)
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1       252G   41G  201G  17% /
/: 198.4 GiB (213031256064 bytes) trimmed on /dev/vda1

# 1. 安装 restic 并自更新至官方最新稳定版二进制文件
sudo apt update && sudo apt install -y restic bzip2 postgresql-client
sudo restic self-update

# 2. 创建仅限 root 访问的配置目录与数据库转储暂存目录
sudo mkdir -p /etc/restic /var/backups/db-dumps
sudo chmod 0700 /etc/restic /var/backups/db-dumps

# 3. 生成 48 字节高熵加密密钥文件并锁定为 0400 只读权限
openssl rand -base64 48 | sudo tee /etc/restic/repo.pw > /dev/null
sudo chmod 0400 /etc/restic/repo.pw

# 4. 配置异地 S3 兼容存储桶（或 SFTP）凭证及 Go 内存控制参数
sudo tee /etc/restic/backup.env > /dev/null <<'EOF'
export RESTIC_REPOSITORY="s3:https://s3.eu-central-003.backblazeb2.com/netcup-prod-backups/rs1000-nue"
export RESTIC_PASSWORD_FILE="/etc/restic/repo.pw"
export AWS_ACCESS_KEY_ID="YOUR_S3_KEY_ID"
export AWS_SECRET_ACCESS_KEY="YOUR_S3_APPLICATION_KEY"
export RESTIC_COMPRESSION="auto"
export GOGC=20
export GOMAXPROCS=2
EOF
sudo chmod 0400 /etc/restic/backup.env

# 5. 初始化远端加密 Restic 仓库
sudo bash -c 'source /etc/restic/backup.env && restic init'

created restic repository 9c4e18f2a0 at s3:https://s3.eu-central-003.backblazeb2.com/netcup-prod-backups/rs1000-nue

sudo tee /usr/local/sbin/netcup-restic-backup > /dev/null <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

# 使用 flock 排他锁防止备份任务重叠执行
LOCKFILE="/var/run/netcup-restic-backup.lock"
exec 9>"${LOCKFILE}"
if ! flock -n 9; then
  echo "Another backup process is currently running. Exiting."
  exit 1
fi

source /etc/restic/backup.env
DUMP_DIR="/var/backups/db-dumps"
mkdir -p "${DUMP_DIR}"
chmod 0700 "${DUMP_DIR}"

echo "==> [1/4] 正在对运行中的 PostgreSQL 容器执行原子级 MVCC 逻辑转储..."
# 遍历所有运行中的 Postgres 容器并导出自定义格式备份
for PG_CONTAINER in $(docker ps --filter "ancestor=postgres:16.4-alpine3.20" --format "{{.Names}}"); do
  echo "  -> 正在导出 PostgreSQL 容器: ${PG_CONTAINER}"
  PG_USER=$(docker inspect "${PG_CONTAINER}" --format '{{range .Config.Env}}{{println .}}{{end}}' | awk -F= '/^POSTGRES_USER=/ {print $2}')
  PG_DB=$(docker inspect "${PG_CONTAINER}" --format '{{range .Config.Env}}{{println .}}{{end}}' | awk -F= '/^POSTGRES_DB=/ {print $2}')
  docker exec "${PG_CONTAINER}" pg_dump -U "${PG_USER:-postgres}" -d "${PG_DB:-postgres}" -Fc --Z=0 \
    > "${DUMP_DIR}/${PG_CONTAINER}_${PG_DB}.pgdump.tmp"
  mv "${DUMP_DIR}/${PG_CONTAINER}_${PG_DB}.pgdump.tmp" "${DUMP_DIR}/${PG_CONTAINER}_${PG_DB}.pgdump"
done

echo "==> [2/4] 正在执行客户端加密的 Restic 异地增量备份..."
restic backup \
  /etc \
  /opt/stacks \
  /var/backups/db-dumps \
  /home \
  /root \
  --tag "automated-daily" \
  --exclude="/var/lib/docker/overlay2" \
  --exclude="**/node_modules" \
  --exclude="**/.cache" \
  --exclude="**/*.log" \
  --one-file-system

echo "==> [3/4] 正在应用 GFS 保留策略（保留 7 天、4 周、6 个月）并清理过期块..."
restic forget \
  --tag "automated-daily" \
  --keep-daily 7 \
  --keep-weekly 4 \
  --keep-monthly 6 \
  --prune

echo "==> [4/4] 正在校验远端仓库索引结构并随机抽检 5% 的数据包..."
restic check --read-data-subset=5%

echo "==> 备份与完整性校验已于 $(date -u +"%Y-%m-%dT%H:%M:%SZ") 全部完成"
EOF

sudo chmod 0700 /usr/local/sbin/netcup-restic-backup

# /etc/systemd/system/restic-backup.service
[Unit]
Description=Encrypted Off-Site Restic Backup & DB Dump for Netcup Server
After=network-online.target docker.service
Wants=network-online.target

[Service]
Type=oneshot
User=root
Nice=19
IOSchedulingClass=idle
IOSchedulingPriority=7
ExecStart=/usr/local/sbin/netcup-restic-backup
StandardOutput=journal
StandardError=journal

sudo tee /etc/systemd/system/restic-backup.timer > /dev/null <<'EOF'
[Unit]
Description=Run Netcup Restic Backup Daily at 03:15 AM with Randomized Jitter

[Timer]
OnCalendar=*-*-* 03:15:00
RandomizedDelaySec=1800
Persistent=true

[Install]
WantedBy=timers.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now restic-backup.timer

# 立即手动触发首次完整备份并查看 systemd 日志输出
sudo systemctl start restic-backup.service
sudo journalctl -u restic-backup.service -n 40 --no-pager
sudo systemctl list-timers restic-backup.timer

NEXT                        LEFT          LAST                        PASSED       UNIT                ACTIVATES
Thu 2026-11-05 03:28:14 CET 6h left       Wed 2026-11-04 20:45:10 CET 12s ago      restic-backup.timer restic-backup.service

# 1. 列出远端存储仓库中的所有加密快照记录
sudo bash -c 'source /etc/restic/backup.env && restic snapshots'

# 2. 从最新快照 (latest) 中将 /opt/stacks 与 /var/backups/db-dumps 选择性恢复至 /tmp/restore-drill
sudo mkdir -p /tmp/restore-drill
sudo bash -c 'source /etc/restic/backup.env && restic restore latest --target /tmp/restore-drill --include /opt/stacks --include /var/backups/db-dumps'

# 3. 校验恢复出的 PostgreSQL 自定义转储归档目录结构 (TOC) 完整性
pg_restore --list /tmp/restore-drill/var/backups/db-dumps/*.pgdump | head -n 20

# 4. 演练验证完毕后清理临时恢复目录
sudo rm -rf /tmp/restore-drill

ID        Time                 Host         Tags             Paths
------------------------------------------------------------------------------------
a81f39c2  2026-11-04 20:45:02  netcup-rs1   automated-daily  /etc, /opt/stacks, /var/backups/db-dumps

; Archive created at 2026-11-04 20:45:00 UTC
;     dbname: prod_app
;     TOC Entries: 48