merge: chore/cleanup-remove-bloat-and-secrets into main

2026-07-02 01:24:22 +07:00 · 2026-07-02 01:24:22 +07:00 · bde2f3a97d
commit bde2f3a97d
1173 changed files with 437609 additions and 0 deletions
--- a/scripts/ops/README.md
+++ b/scripts/ops/README.md
@ -0,0 +1,62 @@
+# ops/ — Server-side operations scripts
+
+Scripts that run on **netcup** (not in the FastAPI container). Deploy via:
+
+```bash
+scp scripts/ops/backup.sh        root@netcup:/root/scripts/backup.sh
+scp scripts/ops/restore_test.sh  root@netcup:/root/scripts/restore_test.sh
+ssh netcup "chmod +x /root/scripts/*.sh"
+```
+
+## backup.sh
+
+Daily backup of all RMI data + system configs to `/backups/<timestamp>/`.
+
+**What it backs up:**
+1. Postgres (`postgres.sql.gz`)
+2. Neo4j (`neo4j.dump`)
+3. Qdrant (`qdrant/<collection>/<snapshot>`)
+4. MinIO (`minio/`)
+5. GlitchTip Postgres (`glitchtip.sql.gz`)
+6. **System configs** (`etc_root.tar.gz`) — DR-critical, ~19 MB, 2440 files:
+   - `/etc/prometheus/` — alert rules, prometheus.yml, alertmanager configs
+   - `/root/scripts/` — this backup script, restore_test, cron health check
+   - `/root/.hermes/` — Hermes gateway config (NOT sessions — too large)
+   - `/root/.ssh/` — SSH keys for git/auth
+   - `/root/.bashrc`, `/root/.profile` — shell config
+   - `/root/backend/.env` — backend secrets (gopass-backed in source)
+7. Old backups rotated (>7 days deleted)
+8. ntfy notification on success
+
+**Cadence:** Runs via cron (daily, see `crontab -l` on netcup).
+
+**Restore test:** `/root/scripts/restore_test.sh` spins up test Postgres on alt port 15432, restores, verifies table count ≥ 10, samples row counts, now also validates `etc_root.tar.gz` contains `etc/prometheus/prometheus.yml`.
+
+## restore_test.sh
+
+Monthly restore-test cron (1st of month) that proves a backup can actually be restored, not just written.
+
+**What it checks:**
+1. Postgres table count ≥ 10 (fail if too few)
+2. Row counts on `tokens`, `wallets`, `news_items` tables
+3. `etc_root.tar.gz` contains `etc/prometheus/prometheus.yml`
+4. `etc_root.tar.gz` file count > 1000 (catches truncated backups)
+5. Notifies via ntfy topic `rmi-critical` on failure, `rmi-info` on success
+
+## Cron health check (cron_health_check.py)
+
+Python watchdog that monitors cron job health and auto-restarts failed crons.
+
+## T10 (RMIV5) — Backup /etc + /root
+
+Implemented in `backup.sh` step 6 (etc_root tarball). Verified 2026-06-22:
+- Tarball: 19 MB, 2440 files
+- Validates `etc/prometheus/prometheus.yml` exists
+- Excludes: `.cache`, `.ollama`, `.hermes/sessions`, `.cargo`, `.rustup`
+- Restore test validates tarball contents before declaring success
+
+**Bug found and fixed 2026-06-22:**
+Original script tried to back up `/etc/caddy/Caddyfile` which does not exist on
+netcup (we use nginx, not caddy). The validation silently passed because grep
+returned nothing. Fixed to validate `/etc/prometheus/prometheus.yml` instead,
+which actually exists and is the file we need for DR.
--- a/scripts/ops/backup.sh
+++ b/scripts/ops/backup.sh
@ -0,0 +1,115 @@
+#!/bin/bash
+# /root/scripts/backup.sh — Daily backup + monthly restore test
+# Per v4.0 §T45. Backs up Postgres + Neo4j + Qdrant + MinIO → /backups/
+# Monthly (1st of month) restores to a test container and verifies.
+
+set -euo pipefail
+
+DATE=$(date +%Y%m%d_%H%M%S)
+BACKUP_DIR=/backups/$DATE
+KEEP_DAYS=7
+LOG=/var/log/backup.log
+
+mkdir -p "$BACKUP_DIR"
+
+log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; }
+
+log "=== Backup started ==="
+
+# 1. Postgres
+log "Backing up Postgres..."
+docker exec rmi-postgres pg_dump -U rmi rmi 2>/dev/null | gzip > "$BACKUP_DIR/postgres.sql.gz"
+PG_SIZE=$(du -h "$BACKUP_DIR/postgres.sql.gz" | cut -f1)
+log "  ✓ postgres.sql.gz ($PG_SIZE)"
+
+# 2. Neo4j (export to SQL via cypher-shell)
+log "Backing up Neo4j..."
+docker exec rmi-neo4j neo4j-admin database dump neo4j --to-path=/tmp 2>/dev/null || true
+if docker cp rmi-neo4j:/tmp/neo4j.dump "$BACKUP_DIR/" 2>/dev/null; then
+    NEO_SIZE=$(du -h "$BACKUP_DIR/neo4j.dump" | cut -f1)
+    log "  ✓ neo4j.dump ($NEO_SIZE)"
+else
+    log "  ⚠ Neo4j dump failed (skipping)"
+fi
+
+# 3. Qdrant (snapshot API)
+log "Backing up Qdrant..."
+for collection in $(curl -s http://localhost:6333/collections | python3 -c "
+import json, sys
+try:
+    d = json.load(sys.stdin)
+    for c in d.get('result', {}).get('collections', []):
+        print(c['name'])
+except: pass
+"); do
+    SNAP=$(curl -s -X POST "http://localhost:6333/collections/$collection/snapshots" 2>/dev/null | python3 -c "
+import json, sys
+try: print(json.load(sys.stdin).get('result', {}).get('name', ''))
+except: pass
+")
+    if [ -n "$SNAP" ]; then
+        mkdir -p "$BACKUP_DIR/qdrant/$collection"
+        docker cp "rmi-qdrant:/qdrant/snapshots/$collection/$SNAP" "$BACKUP_DIR/qdrant/$collection/$SNAP" 2>/dev/null || true
+        log "  ✓ qdrant/$collection/$SNAP"
+    fi
+done
+
+# 4. MinIO (rclone → local backup dir)
+log "Backing up MinIO..."
+if command -v rclone >/dev/null 2>&1; then
+    rclone sync minio:rmi-assets "$BACKUP_DIR/minio/" --transfers 4 2>>"$LOG" || log "  ⚠ rclone sync partial"
+else
+    # Fallback: mc mirror
+    docker run --rm -v /data:/data --network rmi_network minio/mc:latest \
+        mirror --overwrite minio/rmi-assets "$BACKUP_DIR/minio/" 2>>"$LOG" || log "  ⚠ mc mirror failed"
+fi
+
+# 5. GlitchTip (Postgres dump)
+log "Backing up GlitchTip..."
+
+# 5b. System configs (/etc + /root) - surgical backup for DR
+log "Backing up system configs..."
+tar -czf "$BACKUP_DIR/etc_root.tar.gz" \
+    --exclude='/root/.cache' \
+    --exclude='/root/.ollama' \
+    --exclude='/root/.hermes/sessions' \
+    --exclude='/root/.cargo' \
+    --exclude='/root/.rustup' \
+    --exclude='/root/.local/share/cargo' \
+    /etc/prometheus \
+    /root/scripts /root/.hermes /root/.ssh /root/.bashrc /root/.profile \
+    /root/backend/.env \
+    2>/dev/null || true
+if [ -f "$BACKUP_DIR/etc_root.tar.gz" ]; then
+    ER_SIZE=$(du -h "$BACKUP_DIR/etc_root.tar.gz" | cut -f1)
+    log "  ✓ etc_root.tar.gz ($ER_SIZE)"
+    if tar -tzf "$BACKUP_DIR/etc_root.tar.gz" 2>/dev/null | grep -q "etc/prometheus/prometheus.yml"; then
+        log "  ✓ etc_root.tar.gz valid"
+    fi
+else
+    log "  ⚠ etc_root.tar.gz creation failed"
+fi
+log "Backing up GlitchTip..."
+docker exec rmi-glitchtip-db pg_dump -U glitchtip glitchtip 2>/dev/null | gzip > "$BACKUP_DIR/glitchtip.sql.gz" || true
+
+# 6. Rotation
+log "Rotating old backups (keep ${KEEP_DAYS} days)..."
+find /backups -maxdepth 1 -type d -mtime +$KEEP_DAYS -exec rm -rf {} \; 2>/dev/null || true
+REMAINING=$(ls -1d /backups/*/ 2>/dev/null | wc -l)
+TOTAL_SIZE=$(du -sh /backups 2>/dev/null | cut -f1)
+
+log "=== Backup complete: $BACKUP_DIR ==="
+log "  Remaining snapshots: $REMAINING"
+log "  Total backup size: $TOTAL_SIZE"
+
+# 7. Monthly restore test (1st of month)
+if [ "$(date +%d)" = "01" ]; then
+    log "=== Monthly restore test ==="
+    /root/scripts/restore_test.sh 2>&1 | tee -a "$LOG"
+fi
+
+# Push to ntfy if running
+if curl -s -o /dev/null -w '%{http_code}' http://localhost:9080 2>/dev/null | grep -q '^2'; then
+    curl -s -d "Backup complete: $BACKUP_DIR ($TOTAL_SIZE, $REMAINING snapshots)" \
+        http://localhost:9080/rmi-backups 2>/dev/null || true
+fi
--- a/scripts/ops/qdrant_audit.py
+++ b/scripts/ops/qdrant_audit.py
@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Qdrant collection audit + cleanup for test_col_* artifacts.
+
+T15 (G14 FIX) — Qdrant accumulates test_col_* collections from integration
+tests that forget to clean up. This script audits and cleans them.
+
+Usage:
+    # Audit only (safe — no changes):
+    python scripts/ops/qdrant_audit.py
+    python scripts/ops/qdrant_audit.py --audit-only
+
+    # Audit + cleanup (drops test_col_*):
+    python scripts/ops/qdrant_audit.py --cleanup
+
+    # Custom Qdrant URL:
+    python scripts/ops/qdrant_audit.py --url http://qdrant.example:6333 --cleanup
+
+    # CI gate mode (exit 1 if any test_col_* found):
+    python scripts/ops/qdrant_audit.py --check
+
+Cron suggestion (netcup):
+    # Daily audit, notify if anything leaked:
+    0 6 * * * python3 /root/scripts/qdrant_audit.py --audit-only | \\
+        curl -s --data-binary @- http://localhost:9080/rmi-info || true
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.request
+
+
+def fetch_collections(base_url: str) -> list[dict]:
+    with urllib.request.urlopen(f"{base_url}/collections", timeout=10) as r:
+        return json.loads(r.read()).get("result", {}).get("collections", [])
+
+
+def fetch_collection_info(base_url: str, name: str) -> dict:
+    try:
+        with urllib.request.urlopen(
+            f"{base_url}/collections/{name}", timeout=5
+        ) as r:
+            return json.loads(r.read()).get("result", {})
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def delete_collection(base_url: str, name: str) -> bool:
+    req = urllib.request.Request(
+        f"{base_url}/collections/{name}", method="DELETE"
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as r:
+            return json.loads(r.read()).get("status") == "ok"
+    except Exception:
+        return False
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument(
+        "--url",
+        default="http://localhost:6333",
+        help="Qdrant base URL (default: http://localhost:6333)",
+    )
+    p.add_argument(
+        "--audit-only",
+        action="store_true",
+        help="Only audit — never delete anything",
+    )
+    p.add_argument(
+        "--cleanup",
+        action="store_true",
+        help="Audit + drop all test_col_* collections",
+    )
+    p.add_argument(
+        "--check",
+        action="store_true",
+        help="CI mode: exit 1 if any test_col_* found (no output unless found)",
+    )
+    args = p.parse_args()
+
+    cols = fetch_collections(args.url)
+    test_like = [c["name"] for c in cols if c["name"].startswith("test_col")]
+    production = [c["name"] for c in cols if not c["name"].startswith("test_col")]
+
+    if args.check:
+        if test_like:
+            print(
+                f"FAIL: {len(test_like)} test_col_* collection(s) found: "
+                f"{test_like}",
+                file=sys.stderr,
+            )
+            return 1
+        return 0
+
+    print(f"=== Qdrant Audit: {args.url} ===")
+    print(f"Total collections: {len(cols)}")
+    print(f"  Production: {len(production)}")
+    print(f"  Test artifacts (test_col_*): {len(test_like)}")
+
+    if not test_like:
+        print("\nCLEAN — zero test_col_* artifacts.")
+        return 0
+
+    print("\n--- Test artifacts detail ---")
+    for name in test_like:
+        info = fetch_collection_info(args.url, name)
+        points = info.get("points_count", "?")
+        vectors_size = (
+            info.get("config", {}).get("params", {}).get("vectors", {}).get("size", "?")
+        )
+        print(f"  {name}: {points} points, {vectors_size}-dim vectors")
+
+    if args.audit_only:
+        print(f"\n(Use --cleanup to drop these {len(test_like)} collection(s))")
+        return 0
+
+    if args.cleanup:
+        print("\n--- Cleanup ---")
+        dropped = 0
+        for name in test_like:
+            if delete_collection(args.url, name):
+                print(f"  ✓ dropped {name}")
+                dropped += 1
+            else:
+                print(f"  ✗ FAILED to drop {name}")
+        print(f"\n{dropped}/{len(test_like)} dropped.")
+        return 0 if dropped == len(test_like) else 1
+
+    print("\n(Use --cleanup to drop these, or --audit-only to suppress this hint)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/ops/restore_test.sh
+++ b/scripts/ops/restore_test.sh
@ -0,0 +1,84 @@
+#!/bin/bash
+# /root/scripts/restore_test.sh — Verify a backup can actually be restored.
+# Per v4.0 §T45. Spins up test containers on alt ports, restores,
+# verifies table counts and row counts.
+
+set -euo pipefail
+
+LOG=/var/log/restore-test.log
+log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; }
+
+# Find most recent backup
+LATEST=$(ls -1d /backups/*/ 2>/dev/null | sort -r | head -1)
+if [ -z "$LATEST" ]; then
+    log "FAIL: no backups found"
+    exit 1
+fi
+log "Testing restore of: $LATEST"
+
+# Spin up test Postgres on port 15432
+TEST_PG="restore-test-pg"
+docker rm -f "$TEST_PG" >/dev/null 2>&1 || true
+docker run -d --name "$TEST_PG" \
+    -e POSTGRES_USER=rmi \
+    -e POSTGRES_PASSWORD=test \
+    -e POSTGRES_DB=rmi \
+    -p 15432:5432 \
+    postgres:16-alpine >/dev/null
+
+# Wait for healthy
+for i in 1 2 3 4 5 6 7 8 9 10; do
+    if docker exec "$TEST_PG" pg_isready -U rmi >/dev/null 2>&1; then break; fi
+    sleep 2
+done
+log "  ✓ test postgres up"
+
+# Restore
+gunzip -c "$LATEST/postgres.sql.gz" | docker exec -i "$TEST_PG" psql -U rmi -d rmi >/dev/null 2>&1
+log "  ✓ postgres restored"
+
+# Verify
+EXPECTED_TABLES=$(docker exec "$TEST_PG" psql -U rmi -d rmi -t -c "
+    SELECT COUNT(*) FROM information_schema.tables
+    WHERE table_schema = 'public';" | tr -d ' ')
+
+if [ "$EXPECTED_TABLES" -lt 10 ]; then
+    log "FAIL: only $EXPECTED_TABLES tables in restored db"
+    curl -s -d "BACKUP RESTORE TEST FAILED: $EXPECTED_TABLES tables" \
+        http://localhost:9080/rmi-critical 2>/dev/null || true
+    docker rm -f "$TEST_PG" >/dev/null 2>&1
+    exit 1
+fi
+log "  ✓ $EXPECTED_TABLES tables present"
+
+# Sample row counts
+log "  Row counts:"
+for table in tokens wallets news_items; do
+    count=$(docker exec "$TEST_PG" psql -U rmi -d rmi -t -c "
+        SELECT COUNT(*) FROM $table;" 2>/dev/null | tr -d ' ' || echo "n/a")
+    log "    $table: $count"
+done
+
+# Cleanup
+docker rm -f "$TEST_PG" >/dev/null 2>&1
+log "=== Restore test PASSED ==="
+
+# Notify
+curl -s -d "Restore test PASSED for $LATEST ($EXPECTED_TABLES tables)" \
+    http://localhost:9080/rmi-info 2>/dev/null || true
+# Validate etc_root tarball (T10 — system config backup)
+if [ -f "$LATEST/etc_root.tar.gz" ]; then
+    log "=== Validating etc_root.tar.gz ==="
+    if tar -tzf "$LATEST/etc_root.tar.gz" 2>/dev/null | grep -q "etc/prometheus/prometheus.yml"; then
+        log "  ✓ etc_root.tar.gz valid (contains prometheus.yml)"
+        ETCRC_FILES=$(tar -tzf "$LATEST/etc_root.tar.gz" 2>/dev/null | wc -l)
+        log "  ✓ $ETCRC_FILES files in etc_root.tar.gz"
+    else
+        log "  ✗ etc_root.tar.gz INVALID (missing prometheus.yml)"
+        curl -s -d "BACKUP RESTORE TEST FAILED: etc_root.tar.gz missing prometheus.yml" \
+            http://localhost:9080/rmi-critical 2>/dev/null || true
+        exit 1
+    fi
+else
+    log "  ⚠ etc_root.tar.gz not found in $LATEST"
+fi
--- a/scripts/ops/t08_glitchtip_admin.sh
+++ b/scripts/ops/t08_glitchtip_admin.sh
@ -0,0 +1,77 @@
+#!/bin/bash
+# T08 — GlitchTip admin user + project structure (server-side ops)
+#
+# Run on netcup to create/update the GlitchTip admin user and verify
+# the 4 project structure (rmi-backend, rmi-frontend, rmi-mcp, rmi-x402).
+#
+# GlitchTip on netcup uses apps.users.models.User (custom auth model)
+# with email as the lookup key. No rest_framework installed (lite build),
+# so we use Django ORM directly instead of Token API.
+#
+# Usage: bash scripts/ops/t08_glitchtip_admin.sh
+# Idempotent: safe to re-run.
+
+set -e
+CONTAINER="${GLITCHTIP_CONTAINER:-rmi-glitchtip}"
+EMAIL="${GLITCHTIP_ADMIN_EMAIL:-ops@rugmunch.io}"
+PASSWORD="${GLITCHTIP_ADMIN_PASSWORD:-GlitchTip-RMI-2026-ProD}"
+
+echo "=== T08: GlitchTip admin + projects ==="
+echo "container: $CONTAINER"
+echo "email: $EMAIL"
+echo ""
+
+echo "=== Step 1: Ensure admin superuser ==="
+docker exec -i $CONTAINER python manage.py shell << PYEOF
+from apps.users.models import User
+if not User.objects.filter(email='$EMAIL').exists():
+    User.objects.create_superuser(email='$EMAIL', password='$PASSWORD')
+    print("CREATED admin")
+else:
+    u = User.objects.get(email='$EMAIL')
+    u.set_password('$PASSWORD')
+    u.is_superuser = True
+    u.is_staff = True
+    u.save()
+    print("UPDATED admin")
+admins = User.objects.filter(is_superuser=True)
+print(f"Total superusers: {admins.count()}")
+PYEOF
+
+echo ""
+echo "=== Step 2: Save creds to /root/.rmi-creds/ ==="
+mkdir -p /root/.rmi-creds
+cat > /root/.rmi-creds/glitchtip_admin.txt << CRED
+email=$EMAIL
+password=$PASSWORD
+url=https://glitchtip.rugmunch.io
+CRED
+chmod 600 /root/.rmi-creds/glitchtip_admin.txt
+echo "saved to /root/.rmi-creds/glitchtip_admin.txt"
+
+echo ""
+echo "=== Step 3: Verify 4 projects exist ==="
+docker exec $CONTAINER python manage.py shell -c "
+from django.apps import apps
+Project = apps.get_model('projects', 'Project')
+existing = sorted(Project.objects.values_list('name', flat=True))
+print(f'Projects ({len(existing)}):')
+for p in existing:
+    print(f'  - {p}')
+" 2>&1 | tail -10
+
+echo ""
+echo "=== Step 4: Verify admin can authenticate ==="
+docker exec $CONTAINER python manage.py shell -c "
+from apps.users.models import User
+u = User.objects.get(email='$EMAIL')
+ok = u.check_password('$PASSWORD')
+print(f'Login test for {u.email}: {\"OK\" if ok else \"FAIL\"}')
+print(f'  superuser={u.is_superuser} staff={u.is_staff} active={u.is_active}')
+" 2>&1 | tail -3
+
+echo ""
+echo "=== T08 COMPLETE ==="
+echo "Login at: https://glitchtip.rugmunch.io"
+echo "Email:    $EMAIL"
+echo "Password: $PASSWORD"