felhom.eu/documentation/audits/day-2026-10-08/r232/backup-config.sh.after

#!/bin/bash
# Dooplex Cluster Backup Configuration
# Source this file in backup scripts: source /opt/backup/backup-config.sh

# ============================================================================
# BACKUP DESTINATIONS
# ============================================================================
export BACKUP_BASE="/mnt/5_hdd/backup"
export BACKUP_K3S="${BACKUP_BASE}/k3s"
export BACKUP_SECRETS="${BACKUP_BASE}/secrets"
export BACKUP_MANIFESTS="${BACKUP_BASE}/homelab-manifests"
# NOT under BACKUP_BASE. DATA_SOURCE_DIR moved to /mnt/5_hdd/data in the
# 2026-08-14 migration off the failed 4_hdd, so leaving this repo under
# BACKUP_BASE (also 5_hdd) would put the backup on the same physical disk as
# its source -- protection against accidental deletion, none against loss of
# sda1. 1_hdd holds no Longhorn replicas and only serves Plex reads, so backup
# writes do not contend with live volume I/O.
export BACKUP_DATA="/mnt/1_hdd/backup/data"
export BACKUP_LONGHORN="${BACKUP_BASE}/longhorn-pvc"
export BACKUP_LOGS="${BACKUP_BASE}/logs"

# ============================================================================
# RESTIC REPOSITORIES (each category has its own repo for flexibility)
# ============================================================================
export RESTIC_REPO_K3S="${BACKUP_K3S}/restic-repo"
export RESTIC_REPO_SECRETS="${BACKUP_SECRETS}/restic-repo"
export RESTIC_REPO_DATA="${BACKUP_DATA}/restic-repo"
export BACKUP_POSTGRESQL="${BACKUP_BASE}/postgresql"
export POSTGRESQL_DUMP_DIR="${BACKUP_POSTGRESQL}/dumps"
export RESTIC_REPO_POSTGRESQL="${BACKUP_POSTGRESQL}/restic-repo"

# ============================================================================
# RESTIC PASSWORD (change this!)
# Store in /etc/backup/restic-password or set RESTIC_PASSWORD_FILE
# ============================================================================
export RESTIC_PASSWORD_FILE="/etc/backup/restic-password"

# ============================================================================
# RETENTION POLICY
# ============================================================================
export RETENTION_KEEP_LAST=7
export RETENTION_KEEP_DAILY=7
export RETENTION_KEEP_WEEKLY=4
export RETENTION_KEEP_MONTHLY=6

# ============================================================================
# SOURCE DIRECTORIES
# ============================================================================
export K3S_SERVER_DIR="/var/lib/rancher/k3s/server"
export K3S_CONFIG_DIR="/etc/rancher/k3s"
export DATA_SOURCE_DIR="/mnt/5_hdd/data"

# Claude Code auto-memory store (R-229, 2026-08-06). Rides in the User Data component because it is
# small, exists on this host only, and is in NO git repository -- /mnt/5_hdd/felhom.eu/git is not a
# repo, so nothing else preserves it. BACKED UP, NOT COMMITTED: it is auto-written and may name
# hosts and paths that the project's secrets rule keeps out of committed files.
# CAVEAT: BACKUP_BASE is on the SAME physical disk (/mnt/5_hdd) as this source, so this protects
# against accidental deletion, NOT against loss of sda1.
export CLAUDE_MEMORY_DIR="/mnt/5_hdd/felhom.eu/git/.claude-memory"

# ============================================================================
# EXCLUDES
# ============================================================================
export DATA_EXCLUDES=(
    "*.tmp"
    "*.temp"
    "*.cache"
    "**/cache/**"
    "**/Cache/**"
    "**/.cache/**"
    "**/node_modules/**"
    "**/__pycache__/**"
    "**/Thumbs.db"
    "**/.DS_Store"
)

# ============================================================================
# NOTIFICATION (optional - configure as needed)
# ============================================================================
export NOTIFY_ON_FAILURE="true"
# export NOTIFY_WEBHOOK_URL="https://your-webhook-url"
# R-232 (a), 2026-10-08 (operator yes in chat): a failed run is MAILED through the project's existing mail path
# (Resend, the same API the CI failure mail uses) to the operator. The key is stored out-of-band, root-only, in
# NOTIFY_RESEND_KEY_FILE (copied from the k3s Secret felhom-system/resend-api); it is never printed.
export NOTIFY_RESEND_KEY_FILE="/etc/backup/resend-api-key"
export NOTIFY_MAIL_FROM="DooPlex backup <monitoring@felhom.eu>"
export NOTIFY_MAIL_TO="admin@felhom.eu"

# ============================================================================
# HELPER FUNCTIONS
# ============================================================================

log() {
    local level="$1"
    shift
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "${BACKUP_LOGS}/backup.log"
}

log_info() { log "INFO" "$@"; }
log_warn() { log "WARN" "$@"; }
log_error() { log "ERROR" "$@"; }

check_restic() {
    if ! command -v restic &> /dev/null; then
        log_error "restic is not installed. Install with: apt install restic"
        exit 1
    fi
}

check_kubectl() {
    if ! command -v kubectl &> /dev/null; then
        log_error "kubectl is not installed"
        exit 1
    fi
}

ensure_dirs() {
    mkdir -p "${BACKUP_K3S}" "${BACKUP_SECRETS}" "${BACKUP_MANIFESTS}" \
             "${BACKUP_DATA}" "${BACKUP_LONGHORN}" "${BACKUP_LOGS}" "${BACKUP_POSTGRESQL}"
}

init_restic_repo() {
    local repo="$1"
    if [ ! -d "${repo}" ]; then
        log_info "Initializing restic repository: ${repo}"
        restic -r "${repo}" init
    fi
}

apply_retention() {
    local repo="$1"
    log_info "Applying retention policy to ${repo}"
    restic -r "${repo}" forget \
        --keep-last ${RETENTION_KEEP_LAST} \
        --keep-daily ${RETENTION_KEEP_DAILY} \
        --keep-weekly ${RETENTION_KEEP_WEEKLY} \
        --keep-monthly ${RETENTION_KEEP_MONTHLY} \
        --prune
}

notify_failure() {
    local message="$1"
    if [ "${NOTIFY_ON_FAILURE}" = "true" ] && [ -n "${NOTIFY_WEBHOOK_URL}" ]; then
        curl -s -X POST -H "Content-Type: application/json" \
            -d "{\"text\": \"🚨 Backup Failed: ${message}\"}" \
            "${NOTIFY_WEBHOOK_URL}" || true
    fi
    # R-232 (a): the mail. Never fails the caller (a broken mail must not change a backup's exit code); logs its outcome.
    if [ "${NOTIFY_ON_FAILURE}" = "true" ] && [ -r "${NOTIFY_RESEND_KEY_FILE}" ]; then
        if NOTIFY_MSG="${message}" NOTIFY_HOST="$(hostname)" NOTIFY_LOG="${BACKUP_LOGS}/backup.log" python3 - <<'PY'
import json, os, sys, urllib.error, urllib.request
key = open(os.environ["NOTIFY_RESEND_KEY_FILE"]).read().strip()
msg, host = os.environ.get("NOTIFY_MSG", ""), os.environ.get("NOTIFY_HOST", "?")
body = json.dumps({
    "from": os.environ["NOTIFY_MAIL_FROM"], "to": [os.environ["NOTIFY_MAIL_TO"]],
    "subject": "[DooPlex backup] FAILED: %s" % msg,
    "text": "DooPlex's backup reported a failure.\n\nHost   : %s\nFailure: %s\nLog    : %s\n\n"
            "See journalctl -u dooplex-backup and the log above. This mail is sent by notify_failure "
            "in /opt/backup/scripts/backup-config.sh (R-232 a).\n" % (host, msg, os.environ.get("NOTIFY_LOG", "")),
}).encode()
req = urllib.request.Request("https://api.resend.com/emails", data=body, method="POST",
    headers={"Authorization": "Bearer %s" % key, "Content-Type": "application/json",
             # Cloudflare fronts api.resend.com and blocks the default Python-urllib agent (error 1010).
             "User-Agent": "dooplex-backup/1.0"})
try:
    with urllib.request.urlopen(req, timeout=30) as r:
        print("notify_failure: mail accepted id=%s" % json.load(r).get("id"))
except urllib.error.HTTPError as e:
    sys.exit("notify_failure: Resend HTTP %s" % e.code)
except Exception as e:
    sys.exit("notify_failure: mail not sent (%s)" % type(e).__name__)
PY
        then log_info "notify_failure: failure mail sent to ${NOTIFY_MAIL_TO}"
        else log_error "notify_failure: the failure mail could NOT be sent"
        fi
    fi
    return 0
}