merge: chore/cleanup-remove-bloat-and-secrets into main
This commit is contained in:
commit
bde2f3a97d
1173 changed files with 437609 additions and 0 deletions
420
app/routers/status_page.py
Normal file
420
app/routers/status_page.py
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
"""
|
||||
RMI Public Status Page
|
||||
=======================
|
||||
Public-facing status page showing health of all RMI services.
|
||||
No authentication required — designed for transparency and trust.
|
||||
|
||||
Monitors:
|
||||
- Backend API health
|
||||
- Redis connectivity
|
||||
- ClickHouse connectivity
|
||||
- MCP discovery endpoint
|
||||
- x402 catalog endpoint
|
||||
- All payment facilitators
|
||||
- Response time metrics
|
||||
- Incident history
|
||||
|
||||
Endpoints:
|
||||
GET /api/v1/status — Full status JSON
|
||||
GET /status — Public HTML status page
|
||||
GET /api/v1/status/summary — One-line summary for badges
|
||||
POST /api/v1/status/incident — Report incident (admin only)
|
||||
GET /api/v1/status/history — Historical uptime data
|
||||
|
||||
Author: RMI Development
|
||||
Date: 2026-06-05
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
logger = logging.getLogger("status_page")
|
||||
|
||||
router = APIRouter(prefix="/api/v1/status", tags=["status-page"])
|
||||
|
||||
|
||||
# ── Health Check Functions ───────────────────────────────────────
|
||||
|
||||
|
||||
async def check_http_service(url: str, timeout: float = 5.0) -> dict[str, Any]:
|
||||
"""Check an HTTP endpoint."""
|
||||
import aiohttp
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
|
||||
async with session.get(url) as resp:
|
||||
latency_ms = (time.monotonic() - start) * 1000
|
||||
if resp.status == 200:
|
||||
return {
|
||||
"status": "operational",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"http_status": resp.status,
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"status": "degraded",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"http_status": resp.status,
|
||||
"error": f"HTTP {resp.status}",
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
latency_ms = (time.monotonic() - start) * 1000
|
||||
return {
|
||||
"status": "down",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"error": str(e)[:200],
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def check_redis() -> dict[str, Any]:
|
||||
"""Check Redis connectivity."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return {
|
||||
"status": "down",
|
||||
"error": "Redis connection unavailable",
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r.ping()
|
||||
latency_ms = (time.monotonic() - start) * 1000
|
||||
# Get memory info
|
||||
info = r.info("memory")
|
||||
used_memory = info.get("used_memory_human", "unknown")
|
||||
|
||||
return {
|
||||
"status": "operational",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"memory_used": used_memory,
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
latency_ms = (time.monotonic() - start) * 1000
|
||||
return {
|
||||
"status": "down",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"error": str(e)[:200],
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def check_clickhouse() -> dict[str, Any]:
|
||||
"""Check ClickHouse connectivity."""
|
||||
import clickhouse_connect
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
ch_host = os.getenv("CH_HOST", "langfuse-clickhouse-1")
|
||||
ch_user = os.getenv("CH_USER", "clickhouse")
|
||||
ch_password = os.getenv("CH_PASSWORD", "REDACTED")
|
||||
|
||||
client = clickhouse_connect.get_client(
|
||||
host=ch_host,
|
||||
port=8123,
|
||||
username=ch_user,
|
||||
password=ch_password,
|
||||
)
|
||||
client.query("SELECT 1")
|
||||
latency_ms = (time.monotonic() - start) * 1000
|
||||
|
||||
# Get label count
|
||||
label_count = client.query("SELECT count() FROM wallet_memory.wallet_labels").result_rows
|
||||
count = label_count[0][0] if label_count else 0
|
||||
|
||||
return {
|
||||
"status": "operational",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"wallet_labels": count,
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
latency_ms = (time.monotonic() - start) * 1000
|
||||
return {
|
||||
"status": "degraded",
|
||||
"latency_ms": round(latency_ms, 1),
|
||||
"error": str(e)[:200],
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
async def check_service(name: str, config: dict) -> dict[str, Any]:
|
||||
"""Check a single service."""
|
||||
check_type = config.get("check_type", "http")
|
||||
|
||||
if check_type == "http":
|
||||
return await check_http_service(config.get("url", ""))
|
||||
elif check_type == "redis":
|
||||
return check_redis()
|
||||
elif check_type == "clickhouse":
|
||||
return check_clickhouse()
|
||||
else:
|
||||
return {"status": "unknown", "error": f"Unknown check type: {check_type}"}
|
||||
|
||||
|
||||
# ── Incident Management ──────────────────────────────────────────
|
||||
|
||||
|
||||
class IncidentReport(BaseModel):
|
||||
service: str
|
||||
severity: str = "minor" # minor, major, critical
|
||||
message: str
|
||||
resolved: bool = False
|
||||
|
||||
|
||||
def get_incidents(limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""Get recent incidents from Redis."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return []
|
||||
|
||||
incidents = r.lrange("rmi:status:incidents", 0, limit - 1)
|
||||
return [json.loads(i) for i in incidents]
|
||||
|
||||
|
||||
def report_incident(service: str, severity: str, message: str, resolved: bool = False):
|
||||
"""Report an incident."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return
|
||||
|
||||
incident = {
|
||||
"id": f"inc:{int(time.time())}",
|
||||
"service": service,
|
||||
"severity": severity,
|
||||
"message": message,
|
||||
"resolved": resolved,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
r.lpush("rmi:status:incidents", json.dumps(incident))
|
||||
r.ltrim("rmi:status:incidents", 0, 99) # Keep last 100
|
||||
|
||||
# Update current status
|
||||
if not resolved:
|
||||
r.set(
|
||||
f"rmi:status:current:{service}",
|
||||
json.dumps(
|
||||
{
|
||||
"severity": severity,
|
||||
"message": message,
|
||||
"since": incident["created_at"],
|
||||
}
|
||||
),
|
||||
)
|
||||
else:
|
||||
r.delete(f"rmi:status:current:{service}")
|
||||
|
||||
|
||||
def get_uptime_history(days: int = 30) -> dict[str, Any]:
|
||||
"""Get uptime history for the past N days."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return {"error": "redis_unavailable"}
|
||||
|
||||
history = {}
|
||||
for service_name in SERVICES:
|
||||
daily_uptime = []
|
||||
for i in range(days):
|
||||
day = (datetime.now(UTC) - timedelta(days=i)).strftime("%Y-%m-%d")
|
||||
key = f"rmi:status:uptime:{service_name}:{day}"
|
||||
data = r.get(key)
|
||||
day_data = json.loads(data) if data else {"uptime_pct": 100.0, "incidents": 0}
|
||||
daily_uptime.append(
|
||||
{
|
||||
"date": day,
|
||||
**day_data,
|
||||
}
|
||||
)
|
||||
history[service_name] = daily_uptime
|
||||
|
||||
return history
|
||||
|
||||
|
||||
# ── Background Monitor ───────────────────────────────────────────
|
||||
|
||||
|
||||
async def run_all_checks():
|
||||
"""Run all health checks and store results."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return
|
||||
|
||||
results = {}
|
||||
overall_status = "operational"
|
||||
|
||||
for name, config in SERVICES.items():
|
||||
try:
|
||||
result = await check_service(name, config)
|
||||
results[name] = result
|
||||
|
||||
# Update overall status
|
||||
if result.get("status") == "down" and config.get("critical"):
|
||||
overall_status = "down"
|
||||
elif result.get("status") == "degraded" and overall_status != "down":
|
||||
overall_status = "degraded"
|
||||
|
||||
# Store individual result
|
||||
r.set(f"rmi:status:service:{name}", json.dumps(result), ex=120)
|
||||
|
||||
except Exception as e:
|
||||
results[name] = {
|
||||
"status": "down",
|
||||
"error": str(e)[:200],
|
||||
"last_check": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
# Store overall status
|
||||
overall = {
|
||||
"status": overall_status,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"services": results,
|
||||
"version": os.getenv("APP_VERSION", "2.0.0"),
|
||||
}
|
||||
r.set("rmi:status:overall", json.dumps(overall), ex=120)
|
||||
|
||||
|
||||
async def status_monitor_loop():
|
||||
"""Run status checks every 30 seconds."""
|
||||
while True:
|
||||
try:
|
||||
await run_all_checks()
|
||||
except Exception as e:
|
||||
logger.error(f"Status monitor error: {e}")
|
||||
import asyncio
|
||||
|
||||
await asyncio.sleep(30)
|
||||
|
||||
|
||||
# ── Endpoints ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_status():
|
||||
"""Get full status for all services."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return JSONResponse(content={"error": "redis_unavailable"})
|
||||
|
||||
# Try cached result
|
||||
cached = r.get("rmi:status:overall")
|
||||
if cached:
|
||||
overall = json.loads(cached)
|
||||
else:
|
||||
# Run checks on demand
|
||||
overall = {
|
||||
"status": "unknown",
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"services": {},
|
||||
}
|
||||
for name, config in SERVICES.items():
|
||||
try:
|
||||
result = await check_service(name, config)
|
||||
overall["services"][name] = result
|
||||
except Exception as e:
|
||||
overall["services"][name] = {
|
||||
"status": "down",
|
||||
"error": str(e)[:200],
|
||||
}
|
||||
|
||||
# Add incidents
|
||||
overall["incidents"] = get_incidents(10)
|
||||
|
||||
# Add facilitator health
|
||||
try:
|
||||
facilitators = r.get("rmi:facilitator:summary")
|
||||
if facilitators:
|
||||
overall["facilitators"] = json.loads(facilitators)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return JSONResponse(content=overall)
|
||||
|
||||
|
||||
@router.get("/summary")
|
||||
async def status_summary():
|
||||
"""Get one-line summary for status badges."""
|
||||
r = get_redis()
|
||||
if not r:
|
||||
return JSONResponse(content={"status": "unknown"})
|
||||
|
||||
cached = r.get("rmi:status:overall")
|
||||
overall = json.loads(cached) if cached else {"status": "unknown"}
|
||||
|
||||
# Count services by status
|
||||
services = overall.get("services", {})
|
||||
operational = sum(1 for s in services.values() if s.get("status") == "operational")
|
||||
degraded = sum(1 for s in services.values() if s.get("status") == "degraded")
|
||||
down = sum(1 for s in services.values() if s.get("status") == "down")
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"status": overall.get("status", "unknown"),
|
||||
"operational": operational,
|
||||
"degraded": degraded,
|
||||
"down": down,
|
||||
"total": len(services),
|
||||
"timestamp": overall.get("timestamp"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/history")
|
||||
async def status_history(days: int = 7):
|
||||
"""Get uptime history."""
|
||||
days = min(days, 90) # Cap at 90 days
|
||||
return JSONResponse(content=get_uptime_history(days))
|
||||
|
||||
|
||||
@router.post("/incident")
|
||||
async def create_incident(incident: IncidentReport, request: Request):
|
||||
"""Report an incident (admin only)."""
|
||||
# Simple admin check
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if not auth.startswith("Bearer "):
|
||||
return JSONResponse(status_code=401, content={"error": "Admin auth required"})
|
||||
|
||||
report_incident(
|
||||
service=incident.service,
|
||||
severity=incident.severity,
|
||||
message=incident.message,
|
||||
resolved=incident.resolved,
|
||||
)
|
||||
|
||||
return JSONResponse(
|
||||
content={
|
||||
"success": True,
|
||||
"message": "Incident reported",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/services")
|
||||
async def list_services():
|
||||
"""List all monitored services."""
|
||||
return JSONResponse(
|
||||
content={
|
||||
"services": {
|
||||
name: {
|
||||
"name": config["name"],
|
||||
"critical": config.get("critical", False),
|
||||
"check_type": config.get("check_type", "http"),
|
||||
}
|
||||
for name, config in SERVICES.items()
|
||||
}
|
||||
}
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue