409 lines
16 KiB
Python
409 lines
16 KiB
Python
"""
|
|
RMI Analytics Engine — Real-Time Metrics & Trend Visualization
|
|
===============================================================
|
|
Comprehensive analytics system for the RugMunch Intelligence Platform.
|
|
|
|
Features:
|
|
• Real-Time Metrics — CPU, memory, requests, errors, latency
|
|
• Time-Series Storage — Redis-backed rolling windows
|
|
• Trend Detection — automatic anomaly detection, trend arrows
|
|
• User Analytics — DAU, MAU, retention, cohort analysis
|
|
• Financial Analytics — revenue, ARPU, MRR, churn
|
|
• Security Analytics — threats blocked, bot traffic, attack patterns
|
|
• Token Analytics — deployment stats, airdrop metrics, holder growth
|
|
• Custom Dashboards — configurable widget layouts
|
|
• Export — CSV, JSON, Prometheus metrics
|
|
|
|
Integrations:
|
|
- Prometheus metrics export
|
|
- Grafana-compatible data format
|
|
- WebSocket real-time streaming
|
|
- ClickHouse for long-term storage
|
|
|
|
Author: RMI Analytics Team
|
|
Date: 2026-05-31
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from dataclasses import asdict, dataclass, field
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger("rmi_analytics")
|
|
|
|
|
|
# ── Data Models ─────────────────────────────────────────────
|
|
|
|
|
|
@dataclass
|
|
class MetricPoint:
|
|
"""Single time-series data point."""
|
|
|
|
timestamp: float
|
|
value: float
|
|
labels: dict[str, str] = field(default_factory=dict)
|
|
|
|
def to_dict(self) -> dict:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass
|
|
class MetricSeries:
|
|
"""Time-series metric with metadata."""
|
|
|
|
name: str
|
|
description: str
|
|
unit: str
|
|
points: list[MetricPoint] = field(default_factory=list)
|
|
|
|
def latest(self) -> float | None:
|
|
return self.points[-1].value if self.points else None
|
|
|
|
def avg(self, n: int = 60) -> float:
|
|
vals = [p.value for p in self.points[-n:]]
|
|
return sum(vals) / len(vals) if vals else 0.0
|
|
|
|
def trend(self, window: int = 10) -> str:
|
|
"""Return trend direction: up, down, flat."""
|
|
if len(self.points) < window * 2:
|
|
return "flat"
|
|
old_avg = sum(p.value for p in self.points[-window * 2 : -window]) / window
|
|
new_avg = sum(p.value for p in self.points[-window:]) / window
|
|
diff = new_avg - old_avg
|
|
if abs(diff) < 0.01 * old_avg:
|
|
return "flat"
|
|
return "up" if diff > 0 else "down"
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"name": self.name,
|
|
"description": self.description,
|
|
"unit": self.unit,
|
|
"latest": self.latest(),
|
|
"avg_1m": self.avg(60),
|
|
"trend": self.trend(),
|
|
"point_count": len(self.points),
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class DashboardWidget:
|
|
"""Dashboard widget configuration."""
|
|
|
|
widget_id: str
|
|
widget_type: str # line, bar, gauge, counter, table, pie
|
|
title: str
|
|
metric_name: str
|
|
width: int = 6 # Grid columns (1-12)
|
|
height: int = 4
|
|
refresh_interval: int = 30 # seconds
|
|
config: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class Dashboard:
|
|
"""Dashboard configuration."""
|
|
|
|
dashboard_id: str
|
|
name: str
|
|
description: str
|
|
widgets: list[DashboardWidget] = field(default_factory=list)
|
|
created_by: str = ""
|
|
is_default: bool = False
|
|
|
|
|
|
# ── Analytics Engine ────────────────────────────────────────
|
|
|
|
|
|
class AnalyticsEngine:
|
|
"""
|
|
Core analytics engine for real-time metrics and trend analysis.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._metrics: dict[str, MetricSeries] = {}
|
|
self._dashboards: dict[str, Dashboard] = {}
|
|
self._ensure_default_dashboards()
|
|
|
|
def _ensure_default_dashboards(self):
|
|
"""Create default system dashboards."""
|
|
# System Health Dashboard
|
|
system_widgets = [
|
|
DashboardWidget("cpu_gauge", "gauge", "CPU Usage", "cpu_percent", 3, 3, 10),
|
|
DashboardWidget("mem_gauge", "gauge", "Memory Usage", "memory_percent", 3, 3, 10),
|
|
DashboardWidget("disk_gauge", "gauge", "Disk Usage", "disk_percent", 3, 3, 10),
|
|
DashboardWidget("req_counter", "counter", "Requests/min", "requests_per_minute", 3, 3, 10),
|
|
DashboardWidget("cpu_line", "line", "CPU History", "cpu_percent", 6, 4, 30),
|
|
DashboardWidget("mem_line", "line", "Memory History", "memory_percent", 6, 4, 30),
|
|
DashboardWidget("latency_line", "line", "Response Latency", "response_time_ms", 6, 4, 30),
|
|
DashboardWidget("error_line", "line", "Error Rate", "error_rate", 6, 4, 30),
|
|
]
|
|
|
|
self._dashboards["system"] = Dashboard(
|
|
dashboard_id="system",
|
|
name="System Health",
|
|
description="Real-time system performance metrics",
|
|
widgets=system_widgets,
|
|
is_default=True,
|
|
)
|
|
|
|
# Financial Dashboard
|
|
financial_widgets = [
|
|
DashboardWidget("revenue_counter", "counter", "Total Revenue", "revenue_usd", 3, 3, 60),
|
|
DashboardWidget("mrr_counter", "counter", "MRR", "mrr_usd", 3, 3, 60),
|
|
DashboardWidget("arpu_counter", "counter", "ARPU", "arpu_usd", 3, 3, 60),
|
|
DashboardWidget("churn_gauge", "gauge", "Churn Rate", "churn_rate", 3, 3, 60),
|
|
DashboardWidget("revenue_line", "line", "Revenue Trend", "revenue_usd", 6, 4, 300),
|
|
DashboardWidget("payments_line", "line", "Payments", "payments_count", 6, 4, 300),
|
|
]
|
|
|
|
self._dashboards["financial"] = Dashboard(
|
|
dashboard_id="financial",
|
|
name="Financial Analytics",
|
|
description="Revenue, payments, and subscription metrics",
|
|
widgets=financial_widgets,
|
|
is_default=True,
|
|
)
|
|
|
|
# Security Dashboard
|
|
security_widgets = [
|
|
DashboardWidget("threats_counter", "counter", "Threats Blocked", "threats_blocked", 3, 3, 30),
|
|
DashboardWidget("bots_counter", "counter", "Bot Requests", "bot_requests", 3, 3, 30),
|
|
DashboardWidget("attacks_counter", "counter", "Attacks", "attacks_detected", 3, 3, 30),
|
|
DashboardWidget("blocked_ips_counter", "counter", "Blocked IPs", "blocked_ips", 3, 3, 30),
|
|
DashboardWidget("threats_pie", "pie", "Threat Types", "threat_types", 6, 4, 60),
|
|
DashboardWidget("attacks_line", "line", "Attack Timeline", "attacks_detected", 6, 4, 60),
|
|
]
|
|
|
|
self._dashboards["security"] = Dashboard(
|
|
dashboard_id="security",
|
|
name="Security Analytics",
|
|
description="Threat detection and security metrics",
|
|
widgets=security_widgets,
|
|
is_default=True,
|
|
)
|
|
|
|
# User Analytics Dashboard
|
|
user_widgets = [
|
|
DashboardWidget("dau_counter", "counter", "DAU", "daily_active_users", 3, 3, 60),
|
|
DashboardWidget("mau_counter", "counter", "MAU", "monthly_active_users", 3, 3, 60),
|
|
DashboardWidget("new_users_counter", "counter", "New Users", "new_users", 3, 3, 60),
|
|
DashboardWidget("retention_gauge", "gauge", "Retention", "retention_rate", 3, 3, 60),
|
|
DashboardWidget("users_line", "line", "User Growth", "total_users", 6, 4, 300),
|
|
DashboardWidget("tiers_pie", "pie", "User Tiers", "users_by_tier", 6, 4, 300),
|
|
]
|
|
|
|
self._dashboards["users"] = Dashboard(
|
|
dashboard_id="users",
|
|
name="User Analytics",
|
|
description="User growth, engagement, and retention",
|
|
widgets=user_widgets,
|
|
is_default=True,
|
|
)
|
|
|
|
# ── Metric Recording ────────────────────────────────────
|
|
|
|
def record_metric(self, name: str, value: float, labels: dict[str, str] | None = None):
|
|
"""Record a metric data point."""
|
|
if name not in self._metrics:
|
|
self._metrics[name] = MetricSeries(
|
|
name=name,
|
|
description=name.replace("_", " ").title(),
|
|
unit="",
|
|
)
|
|
|
|
point = MetricPoint(
|
|
timestamp=time.time(),
|
|
value=value,
|
|
labels=labels or {},
|
|
)
|
|
|
|
self._metrics[name].points.append(point)
|
|
|
|
# Keep only last 10000 points (about 2.7 hours at 1/sec)
|
|
if len(self._metrics[name].points) > 10000:
|
|
self._metrics[name].points = self._metrics[name].points[-10000:]
|
|
|
|
def get_metric(self, name: str) -> MetricSeries | None:
|
|
"""Get metric series by name."""
|
|
return self._metrics.get(name)
|
|
|
|
def get_metric_names(self) -> list[str]:
|
|
"""List all metric names."""
|
|
return list(self._metrics.keys())
|
|
|
|
# ── Dashboard Management ────────────────────────────────
|
|
|
|
def get_dashboard(self, dashboard_id: str) -> Dashboard | None:
|
|
"""Get dashboard by ID."""
|
|
return self._dashboards.get(dashboard_id)
|
|
|
|
def list_dashboards(self) -> list[Dashboard]:
|
|
"""List all dashboards."""
|
|
return list(self._dashboards.values())
|
|
|
|
def create_dashboard(self, name: str, description: str, created_by: str = "") -> Dashboard:
|
|
"""Create a new dashboard."""
|
|
dashboard_id = f"dash_{int(time.time())}_{os.urandom(4).hex()}"
|
|
dashboard = Dashboard(
|
|
dashboard_id=dashboard_id,
|
|
name=name,
|
|
description=description,
|
|
created_by=created_by,
|
|
)
|
|
self._dashboards[dashboard_id] = dashboard
|
|
return dashboard
|
|
|
|
def add_widget(self, dashboard_id: str, widget: DashboardWidget) -> bool:
|
|
"""Add widget to dashboard."""
|
|
dashboard = self._dashboards.get(dashboard_id)
|
|
if not dashboard:
|
|
return False
|
|
dashboard.widgets.append(widget)
|
|
return True
|
|
|
|
# ── Real-Time Data ──────────────────────────────────────
|
|
|
|
def get_dashboard_data(self, dashboard_id: str) -> dict[str, Any]:
|
|
"""Get current data for all widgets in a dashboard."""
|
|
dashboard = self._dashboards.get(dashboard_id)
|
|
if not dashboard:
|
|
return {"error": "Dashboard not found"}
|
|
|
|
widgets_data = []
|
|
for widget in dashboard.widgets:
|
|
metric = self._metrics.get(widget.metric_name)
|
|
data = {
|
|
"widget_id": widget.widget_id,
|
|
"widget_type": widget.widget_type,
|
|
"title": widget.title,
|
|
"metric": metric.to_dict() if metric else {"name": widget.metric_name, "latest": None},
|
|
}
|
|
|
|
# Add historical data for line/bar charts
|
|
if widget.widget_type in ["line", "bar"] and metric:
|
|
# Return last 60 points
|
|
data["history"] = [{"t": p.timestamp, "v": p.value} for p in metric.points[-60:]]
|
|
|
|
widgets_data.append(data)
|
|
|
|
return {
|
|
"dashboard_id": dashboard_id,
|
|
"name": dashboard.name,
|
|
"updated_at": datetime.now(UTC).isoformat(),
|
|
"widgets": widgets_data,
|
|
}
|
|
|
|
# ── Trend Analysis ──────────────────────────────────────
|
|
|
|
def detect_trends(self, metric_name: str, window: int = 60) -> dict[str, Any]:
|
|
"""Detect trends in a metric."""
|
|
metric = self._metrics.get(metric_name)
|
|
if not metric or len(metric.points) < window * 2:
|
|
return {"error": "Insufficient data"}
|
|
|
|
points = metric.points[-window * 2 :]
|
|
half = len(points) // 2
|
|
|
|
first_half = [p.value for p in points[:half]]
|
|
second_half = [p.value for p in points[half:]]
|
|
|
|
first_avg = sum(first_half) / len(first_half)
|
|
second_avg = sum(second_half) / len(second_half)
|
|
|
|
change_pct = ((second_avg - first_avg) / first_avg * 100) if first_avg else 0
|
|
|
|
# Detect anomalies (values outside 2 std dev)
|
|
all_vals = [p.value for p in metric.points[-window:]]
|
|
mean = sum(all_vals) / len(all_vals)
|
|
variance = sum((v - mean) ** 2 for v in all_vals) / len(all_vals)
|
|
std_dev = variance**0.5
|
|
|
|
anomalies = [
|
|
{"timestamp": p.timestamp, "value": p.value}
|
|
for p in metric.points[-window:]
|
|
if abs(p.value - mean) > 2 * std_dev
|
|
]
|
|
|
|
return {
|
|
"metric": metric_name,
|
|
"trend": metric.trend(window),
|
|
"change_percent": round(change_pct, 2),
|
|
"first_period_avg": round(first_avg, 4),
|
|
"second_period_avg": round(second_avg, 4),
|
|
"anomalies_count": len(anomalies),
|
|
"anomalies": anomalies[:5], # Top 5
|
|
}
|
|
|
|
# ── Statistics ───────────────────────────────────────────
|
|
|
|
def get_system_stats(self) -> dict[str, Any]:
|
|
"""Get comprehensive system statistics."""
|
|
return {
|
|
"metrics_tracked": len(self._metrics),
|
|
"dashboards": len(self._dashboards),
|
|
"total_data_points": sum(len(m.points) for m in self._metrics.values()),
|
|
"last_updated": datetime.now(UTC).isoformat(),
|
|
"top_metrics": [
|
|
{"name": name, "points": len(m.points), "latest": m.latest()}
|
|
for name, m in sorted(self._metrics.items(), key=lambda x: len(x[1].points), reverse=True)[:10]
|
|
],
|
|
}
|
|
|
|
# ── Prometheus Export ───────────────────────────────────
|
|
|
|
def to_prometheus(self) -> str:
|
|
"""Export metrics in Prometheus text format."""
|
|
lines = []
|
|
for name, metric in self._metrics.items():
|
|
prom_name = f"rmi_{name}"
|
|
lines.append(f"# HELP {prom_name} {metric.description}")
|
|
lines.append(f"# TYPE {prom_name} gauge")
|
|
|
|
latest = metric.latest()
|
|
if latest is not None:
|
|
labels_str = ", ".join(f'{k}="{v}"' for k, v in metric.points[-1].labels.items())
|
|
if labels_str:
|
|
lines.append(f"{prom_name}{{{labels_str}}} {latest}")
|
|
else:
|
|
lines.append(f"{prom_name} {latest}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
# ── Export ────────────────────────────────────────────
|
|
|
|
def export_metric(self, name: str, format: str = "json") -> Any:
|
|
"""Export metric data."""
|
|
metric = self._metrics.get(name)
|
|
if not metric:
|
|
return None
|
|
|
|
if format == "json":
|
|
return {
|
|
"name": metric.name,
|
|
"description": metric.description,
|
|
"unit": metric.unit,
|
|
"data": [{"timestamp": p.timestamp, "value": p.value, "labels": p.labels} for p in metric.points],
|
|
}
|
|
elif format == "csv":
|
|
lines = ["timestamp,value"]
|
|
for p in metric.points:
|
|
lines.append(f"{p.timestamp},{p.value}")
|
|
return "\n".join(lines)
|
|
|
|
return None
|
|
|
|
|
|
# ── Singleton ─────────────────────────────────────────────────
|
|
|
|
_analytics_instance: AnalyticsEngine | None = None
|
|
|
|
|
|
def get_analytics_engine() -> AnalyticsEngine:
|
|
"""Get or create analytics engine instance."""
|
|
global _analytics_instance
|
|
if _analytics_instance is None:
|
|
_analytics_instance = AnalyticsEngine()
|
|
return _analytics_instance
|