merge: chore/cleanup-remove-bloat-and-secrets into main

scripts/ops/backup.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+# /root/scripts/backup.sh — Daily backup + monthly restore test
+# Per v4.0 §T45. Backs up Postgres + Neo4j + Qdrant + MinIO → /backups/
+# Monthly (1st of month) restores to a test container and verifies.
+
+set -euo pipefail
+
+DATE=$(date +%Y%m%d_%H%M%S)
+BACKUP_DIR=/backups/$DATE
+KEEP_DAYS=7
+LOG=/var/log/backup.log
+
+mkdir -p "$BACKUP_DIR"
+
+log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; }
+
+log "=== Backup started ==="
+
+# 1. Postgres
+log "Backing up Postgres..."
+docker exec rmi-postgres pg_dump -U rmi rmi 2>/dev/null | gzip > "$BACKUP_DIR/postgres.sql.gz"
+PG_SIZE=$(du -h "$BACKUP_DIR/postgres.sql.gz" | cut -f1)
+log "  ✓ postgres.sql.gz ($PG_SIZE)"
+
+# 2. Neo4j (export to SQL via cypher-shell)
+log "Backing up Neo4j..."
+docker exec rmi-neo4j neo4j-admin database dump neo4j --to-path=/tmp 2>/dev/null || true
+if docker cp rmi-neo4j:/tmp/neo4j.dump "$BACKUP_DIR/" 2>/dev/null; then
+    NEO_SIZE=$(du -h "$BACKUP_DIR/neo4j.dump" | cut -f1)
+    log "  ✓ neo4j.dump ($NEO_SIZE)"
+else
+    log "  ⚠ Neo4j dump failed (skipping)"
+fi
+
+# 3. Qdrant (snapshot API)
+log "Backing up Qdrant..."
+for collection in $(curl -s http://localhost:6333/collections | python3 -c "
+import json, sys
+try:
+    d = json.load(sys.stdin)
+    for c in d.get('result', {}).get('collections', []):
+        print(c['name'])
+except: pass
+"); do
+    SNAP=$(curl -s -X POST "http://localhost:6333/collections/$collection/snapshots" 2>/dev/null | python3 -c "
+import json, sys
+try: print(json.load(sys.stdin).get('result', {}).get('name', ''))
+except: pass
+")
+    if [ -n "$SNAP" ]; then
+        mkdir -p "$BACKUP_DIR/qdrant/$collection"
+        docker cp "rmi-qdrant:/qdrant/snapshots/$collection/$SNAP" "$BACKUP_DIR/qdrant/$collection/$SNAP" 2>/dev/null || true
+        log "  ✓ qdrant/$collection/$SNAP"
+    fi
+done
+
+# 4. MinIO (rclone → local backup dir)
+log "Backing up MinIO..."
+if command -v rclone >/dev/null 2>&1; then
+    rclone sync minio:rmi-assets "$BACKUP_DIR/minio/" --transfers 4 2>>"$LOG" || log "  ⚠ rclone sync partial"
+else
+    # Fallback: mc mirror
+    docker run --rm -v /data:/data --network rmi_network minio/mc:latest \
+        mirror --overwrite minio/rmi-assets "$BACKUP_DIR/minio/" 2>>"$LOG" || log "  ⚠ mc mirror failed"
+fi
+
+# 5. GlitchTip (Postgres dump)
+log "Backing up GlitchTip..."
+
+# 5b. System configs (/etc + /root) - surgical backup for DR
+log "Backing up system configs..."
+tar -czf "$BACKUP_DIR/etc_root.tar.gz" \
+    --exclude='/root/.cache' \
+    --exclude='/root/.ollama' \
+    --exclude='/root/.hermes/sessions' \
+    --exclude='/root/.cargo' \
+    --exclude='/root/.rustup' \
+    --exclude='/root/.local/share/cargo' \
+    /etc/prometheus \
+    /root/scripts /root/.hermes /root/.ssh /root/.bashrc /root/.profile \
+    /root/backend/.env \
+    2>/dev/null || true
+if [ -f "$BACKUP_DIR/etc_root.tar.gz" ]; then
+    ER_SIZE=$(du -h "$BACKUP_DIR/etc_root.tar.gz" | cut -f1)
+    log "  ✓ etc_root.tar.gz ($ER_SIZE)"
+    if tar -tzf "$BACKUP_DIR/etc_root.tar.gz" 2>/dev/null | grep -q "etc/prometheus/prometheus.yml"; then
+        log "  ✓ etc_root.tar.gz valid"
+    fi
+else
+    log "  ⚠ etc_root.tar.gz creation failed"
+fi
+log "Backing up GlitchTip..."
+docker exec rmi-glitchtip-db pg_dump -U glitchtip glitchtip 2>/dev/null | gzip > "$BACKUP_DIR/glitchtip.sql.gz" || true
+
+# 6. Rotation
+log "Rotating old backups (keep ${KEEP_DAYS} days)..."
+find /backups -maxdepth 1 -type d -mtime +$KEEP_DAYS -exec rm -rf {} \; 2>/dev/null || true
+REMAINING=$(ls -1d /backups/*/ 2>/dev/null | wc -l)
+TOTAL_SIZE=$(du -sh /backups 2>/dev/null | cut -f1)
+
+log "=== Backup complete: $BACKUP_DIR ==="
+log "  Remaining snapshots: $REMAINING"
+log "  Total backup size: $TOTAL_SIZE"
+
+# 7. Monthly restore test (1st of month)
+if [ "$(date +%d)" = "01" ]; then
+    log "=== Monthly restore test ==="
+    /root/scripts/restore_test.sh 2>&1 | tee -a "$LOG"
+fi
+
+# Push to ntfy if running
+if curl -s -o /dev/null -w '%{http_code}' http://localhost:9080 2>/dev/null | grep -q '^2'; then
+    curl -s -d "Backup complete: $BACKUP_DIR ($TOTAL_SIZE, $REMAINING snapshots)" \
+        http://localhost:9080/rmi-backups 2>/dev/null || true
+fi