115 lines
4.1 KiB
Bash
115 lines
4.1 KiB
Bash
#!/bin/bash
|
|
# /root/scripts/backup.sh — Daily backup + monthly restore test
|
|
# Per v4.0 §T45. Backs up Postgres + Neo4j + Qdrant + MinIO → /backups/
|
|
# Monthly (1st of month) restores to a test container and verifies.
|
|
|
|
set -euo pipefail
|
|
|
|
DATE=$(date +%Y%m%d_%H%M%S)
|
|
BACKUP_DIR=/backups/$DATE
|
|
KEEP_DAYS=7
|
|
LOG=/var/log/backup.log
|
|
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
log() { echo "[$(date +%H:%M:%S)] $*" | tee -a "$LOG"; }
|
|
|
|
log "=== Backup started ==="
|
|
|
|
# 1. Postgres
|
|
log "Backing up Postgres..."
|
|
docker exec rmi-postgres pg_dump -U rmi rmi 2>/dev/null | gzip > "$BACKUP_DIR/postgres.sql.gz"
|
|
PG_SIZE=$(du -h "$BACKUP_DIR/postgres.sql.gz" | cut -f1)
|
|
log " ✓ postgres.sql.gz ($PG_SIZE)"
|
|
|
|
# 2. Neo4j (export to SQL via cypher-shell)
|
|
log "Backing up Neo4j..."
|
|
docker exec rmi-neo4j neo4j-admin database dump neo4j --to-path=/tmp 2>/dev/null || true
|
|
if docker cp rmi-neo4j:/tmp/neo4j.dump "$BACKUP_DIR/" 2>/dev/null; then
|
|
NEO_SIZE=$(du -h "$BACKUP_DIR/neo4j.dump" | cut -f1)
|
|
log " ✓ neo4j.dump ($NEO_SIZE)"
|
|
else
|
|
log " ⚠ Neo4j dump failed (skipping)"
|
|
fi
|
|
|
|
# 3. Qdrant (snapshot API)
|
|
log "Backing up Qdrant..."
|
|
for collection in $(curl -s http://localhost:6333/collections | python3 -c "
|
|
import json, sys
|
|
try:
|
|
d = json.load(sys.stdin)
|
|
for c in d.get('result', {}).get('collections', []):
|
|
print(c['name'])
|
|
except: pass
|
|
"); do
|
|
SNAP=$(curl -s -X POST "http://localhost:6333/collections/$collection/snapshots" 2>/dev/null | python3 -c "
|
|
import json, sys
|
|
try: print(json.load(sys.stdin).get('result', {}).get('name', ''))
|
|
except: pass
|
|
")
|
|
if [ -n "$SNAP" ]; then
|
|
mkdir -p "$BACKUP_DIR/qdrant/$collection"
|
|
docker cp "rmi-qdrant:/qdrant/snapshots/$collection/$SNAP" "$BACKUP_DIR/qdrant/$collection/$SNAP" 2>/dev/null || true
|
|
log " ✓ qdrant/$collection/$SNAP"
|
|
fi
|
|
done
|
|
|
|
# 4. MinIO (rclone → local backup dir)
|
|
log "Backing up MinIO..."
|
|
if command -v rclone >/dev/null 2>&1; then
|
|
rclone sync minio:rmi-assets "$BACKUP_DIR/minio/" --transfers 4 2>>"$LOG" || log " ⚠ rclone sync partial"
|
|
else
|
|
# Fallback: mc mirror
|
|
docker run --rm -v /data:/data --network rmi_network minio/mc:latest \
|
|
mirror --overwrite minio/rmi-assets "$BACKUP_DIR/minio/" 2>>"$LOG" || log " ⚠ mc mirror failed"
|
|
fi
|
|
|
|
# 5. GlitchTip (Postgres dump)
|
|
log "Backing up GlitchTip..."
|
|
|
|
# 5b. System configs (/etc + /root) - surgical backup for DR
|
|
log "Backing up system configs..."
|
|
tar -czf "$BACKUP_DIR/etc_root.tar.gz" \
|
|
--exclude='/root/.cache' \
|
|
--exclude='/root/.ollama' \
|
|
--exclude='/root/.hermes/sessions' \
|
|
--exclude='/root/.cargo' \
|
|
--exclude='/root/.rustup' \
|
|
--exclude='/root/.local/share/cargo' \
|
|
/etc/prometheus \
|
|
/root/scripts /root/.hermes /root/.ssh /root/.bashrc /root/.profile \
|
|
/root/backend/.env \
|
|
2>/dev/null || true
|
|
if [ -f "$BACKUP_DIR/etc_root.tar.gz" ]; then
|
|
ER_SIZE=$(du -h "$BACKUP_DIR/etc_root.tar.gz" | cut -f1)
|
|
log " ✓ etc_root.tar.gz ($ER_SIZE)"
|
|
if tar -tzf "$BACKUP_DIR/etc_root.tar.gz" 2>/dev/null | grep -q "etc/prometheus/prometheus.yml"; then
|
|
log " ✓ etc_root.tar.gz valid"
|
|
fi
|
|
else
|
|
log " ⚠ etc_root.tar.gz creation failed"
|
|
fi
|
|
log "Backing up GlitchTip..."
|
|
docker exec rmi-glitchtip-db pg_dump -U glitchtip glitchtip 2>/dev/null | gzip > "$BACKUP_DIR/glitchtip.sql.gz" || true
|
|
|
|
# 6. Rotation
|
|
log "Rotating old backups (keep ${KEEP_DAYS} days)..."
|
|
find /backups -maxdepth 1 -type d -mtime +$KEEP_DAYS -exec rm -rf {} \; 2>/dev/null || true
|
|
REMAINING=$(ls -1d /backups/*/ 2>/dev/null | wc -l)
|
|
TOTAL_SIZE=$(du -sh /backups 2>/dev/null | cut -f1)
|
|
|
|
log "=== Backup complete: $BACKUP_DIR ==="
|
|
log " Remaining snapshots: $REMAINING"
|
|
log " Total backup size: $TOTAL_SIZE"
|
|
|
|
# 7. Monthly restore test (1st of month)
|
|
if [ "$(date +%d)" = "01" ]; then
|
|
log "=== Monthly restore test ==="
|
|
/root/scripts/restore_test.sh 2>&1 | tee -a "$LOG"
|
|
fi
|
|
|
|
# Push to ntfy if running
|
|
if curl -s -o /dev/null -w '%{http_code}' http://localhost:9080 2>/dev/null | grep -q '^2'; then
|
|
curl -s -d "Backup complete: $BACKUP_DIR ($TOTAL_SIZE, $REMAINING snapshots)" \
|
|
http://localhost:9080/rmi-backups 2>/dev/null || true
|
|
fi
|