|
| 1 | +#!/usr/bin/env bash |
| 2 | +# SPDX-License-Identifier: AGPL-3.0-or-later |
| 3 | +# ============================================================================= |
| 4 | +# Überwachung der Backups (stündlich per Cron auf dem Host, als root) |
| 5 | +# ============================================================================= |
| 6 | +# Meldet per Mail über den mandari-Container (SMTP aus den Systemeinstellungen): |
| 7 | +# - Backup-Container läuft nicht |
| 8 | +# - letzte Sicherung endete mit Fehler |
| 9 | +# - letzte erfolgreiche Sicherung älter als BACKUP_MAX_AGE_HOURS |
| 10 | +# - Integritätsprüfung oder Wiederherstellungstest fehlgeschlagen |
| 11 | +# Solange ein Problem besteht, höchstens eine Mail je 6 Stunden. |
| 12 | +# |
| 13 | +# scripts/backup-watchdog.sh # prüfen (Exit 1 bei Problemen) |
| 14 | +# scripts/backup-watchdog.sh --test # Testmail senden |
| 15 | +# ============================================================================= |
| 16 | +set -uo pipefail |
| 17 | + |
| 18 | +BACKUP_DIR="${BACKUP_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" |
| 19 | + |
| 20 | +env_value() { |
| 21 | + grep -E "^$1=" "$BACKUP_DIR/.env" 2>/dev/null | head -1 | cut -d= -f2- | sed -E 's/^"(.*)"$/\1/' |
| 22 | +} |
| 23 | + |
| 24 | +STATUS_DIR="$(env_value BACKUP_STATUS_DIR)" |
| 25 | +STATUS_DIR="${STATUS_DIR:-./status}" |
| 26 | +case "$STATUS_DIR" in |
| 27 | + /*) ;; |
| 28 | + *) STATUS_DIR="$BACKUP_DIR/${STATUS_DIR#./}" ;; |
| 29 | +esac |
| 30 | +MAX_AGE_HOURS="$(env_value BACKUP_MAX_AGE_HOURS)"; MAX_AGE_HOURS="${MAX_AGE_HOURS:-26}" |
| 31 | +ALERT_TO="$(env_value BACKUP_ALERT_TO)" |
| 32 | +MAIL_CONTAINER="$(env_value BACKUP_MAIL_CONTAINER)"; MAIL_CONTAINER="${MAIL_CONTAINER:-mandari}" |
| 33 | +BACKUP_CONTAINER="$(env_value BACKUP_COMPOSE_PROJECT)"; BACKUP_CONTAINER="${BACKUP_CONTAINER:-mandari-backup}" |
| 34 | +HOST_LABEL="$(env_value BACKUP_HOSTNAME)"; HOST_LABEL="${HOST_LABEL:-$(hostname)}" |
| 35 | +ALERT_STAMP="$STATUS_DIR/.alert-sent" |
| 36 | + |
| 37 | +mkdir -p "$STATUS_DIR" |
| 38 | + |
| 39 | +send_mail() { |
| 40 | + local subject="$1" body="$2" |
| 41 | + if [ -z "$ALERT_TO" ]; then |
| 42 | + echo "BACKUP_ALERT_TO ist nicht gesetzt – keine Mail versendet" >&2 |
| 43 | + return 1 |
| 44 | + fi |
| 45 | + docker exec -e ALERT_SUBJECT="$subject" -e ALERT_BODY="$body" -e ALERT_TO="$ALERT_TO" "$MAIL_CONTAINER" \ |
| 46 | + python manage.py shell -c "import os |
| 47 | +from django.core.mail import send_mail |
| 48 | +send_mail(os.environ['ALERT_SUBJECT'], os.environ['ALERT_BODY'], None, |
| 49 | + [a.strip() for a in os.environ['ALERT_TO'].split(',') if a.strip()])" >/dev/null 2>&1 |
| 50 | +} |
| 51 | + |
| 52 | +if [ "${1:-}" = "--test" ]; then |
| 53 | + if send_mail "[$HOST_LABEL] Backup-Überwachung: Testmail" \ |
| 54 | + "Die Backup-Überwachung auf $HOST_LABEL kann Mails versenden. Es ist keine Aktion nötig."; then |
| 55 | + echo "Testmail an $ALERT_TO versendet" |
| 56 | + exit 0 |
| 57 | + fi |
| 58 | + echo "Testmail fehlgeschlagen" >&2 |
| 59 | + exit 1 |
| 60 | +fi |
| 61 | + |
| 62 | +now=$(date +%s) |
| 63 | +problems=() |
| 64 | + |
| 65 | +running=$(docker inspect -f '{{.State.Running}}' "$BACKUP_CONTAINER" 2>/dev/null || echo "nicht vorhanden") |
| 66 | +[ "$running" = "true" ] || problems+=("Der Backup-Container $BACKUP_CONTAINER läuft nicht (Status: $running).") |
| 67 | + |
| 68 | +if [ -f "$STATUS_DIR/last_success" ]; then |
| 69 | + age=$(( (now - $(stat -c %Y "$STATUS_DIR/last_success")) / 3600 )) |
| 70 | + if [ "$age" -ge "$MAX_AGE_HOURS" ]; then |
| 71 | + problems+=("Die letzte erfolgreiche Sicherung ist $age Stunden alt ($(cat "$STATUS_DIR/last_success")).") |
| 72 | + fi |
| 73 | +else |
| 74 | + [ -f "$STATUS_DIR/.installed" ] || touch "$STATUS_DIR/.installed" |
| 75 | + age=$(( (now - $(stat -c %Y "$STATUS_DIR/.installed")) / 3600 )) |
| 76 | + if [ "$age" -ge "$MAX_AGE_HOURS" ]; then |
| 77 | + problems+=("Seit der Einrichtung vor $age Stunden gab es keine erfolgreiche Sicherung.") |
| 78 | + fi |
| 79 | +fi |
| 80 | + |
| 81 | +if [ -s "$STATUS_DIR/last_error" ] && |
| 82 | + { [ ! -f "$STATUS_DIR/last_success" ] || [ "$STATUS_DIR/last_error" -nt "$STATUS_DIR/last_success" ]; }; then |
| 83 | + problems+=("Die letzte Sicherung endete mit einem Fehler: $(head -c 1500 "$STATUS_DIR/last_error")") |
| 84 | +fi |
| 85 | + |
| 86 | +for marker in check_failed restore_test_failed; do |
| 87 | + if [ -s "$STATUS_DIR/$marker" ]; then |
| 88 | + case "$marker" in |
| 89 | + check_failed) what="Die letzte Integritätsprüfung ist fehlgeschlagen" ;; |
| 90 | + *) what="Der letzte Wiederherstellungstest ist fehlgeschlagen" ;; |
| 91 | + esac |
| 92 | + problems+=("$what: $(head -c 1500 "$STATUS_DIR/$marker")") |
| 93 | + fi |
| 94 | +done |
| 95 | + |
| 96 | +if [ ${#problems[@]} -eq 0 ]; then |
| 97 | + rm -f "$ALERT_STAMP" |
| 98 | + exit 0 |
| 99 | +fi |
| 100 | + |
| 101 | +printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${problems[@]}" |
| 102 | + |
| 103 | +if [ -f "$ALERT_STAMP" ] && [ $(( now - $(stat -c %Y "$ALERT_STAMP") )) -lt 21600 ]; then |
| 104 | + exit 1 |
| 105 | +fi |
| 106 | + |
| 107 | +body="$(printf '%s\n\n' "${problems[@]}") |
| 108 | +Prüfen: |
| 109 | + docker logs --tail 200 $BACKUP_CONTAINER |
| 110 | + docker exec $BACKUP_CONTAINER mandari-backup snapshots |
| 111 | + ls -la $STATUS_DIR |
| 112 | +
|
| 113 | +Anleitung: docs/BACKUP.md im mandari-Repository." |
| 114 | + |
| 115 | +if send_mail "[$HOST_LABEL] Backup-Warnung" "$body"; then |
| 116 | + touch "$ALERT_STAMP" |
| 117 | +else |
| 118 | + echo "Mailversand fehlgeschlagen" >&2 |
| 119 | +fi |
| 120 | +exit 1 |
0 commit comments