diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de522e2..a289bfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ on: # # Phase 1 of AUDIT-2026-Q3.md item P1.6: # - CI was 6/8 decorative (|| true / continue-on-error: true on every job) -# - Now: 2 GATING jobs (build + test) must pass for merge -# - 6 INFORMATIONAL jobs (lint-info, typecheck-info, etc.) report but never gate +# - Now: 3 GATING jobs (build + test + typecheck-gate) must pass for merge +# - 6 INFORMATIONAL jobs (lint-info, typecheck-full-info, etc.) report but never gate # - Forgejo .forgejo/workflows/ci.yml remains the authoritative gate jobs: @@ -61,6 +61,21 @@ jobs: 2>&1 | tail -50 # NO || true — failure GATES merge + typecheck-gate: + name: Typecheck scoped gate (gate) + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v2 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install project editable + run: uv pip install --system -e ".[dev]" + - name: Run scoped mypy gate + run: make mypy-gate + # ────────────────────────── INFORMATIONAL JOBS (fire-and-forget) ────────────────────────── lint-info: @@ -79,8 +94,8 @@ jobs: - name: Run ruff lint (informational) run: ruff check . --statistics --output-format=concise 2>&1 | tail -30 || true - typecheck-info: - name: Typecheck (info only) + typecheck-full-info: + name: Typecheck full codebase (info only) if: always() runs-on: ubuntu-latest continue-on-error: true @@ -93,7 +108,7 @@ jobs: - name: Install mypy run: uv pip install --system mypy - name: Run mypy typecheck (informational) - run: mypy app/ --config-file mypy.ini || true + run: make typecheck || true security-info: name: Security (info only) diff --git a/Makefile b/Makefile index 08de881..152b2f9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev build lint format check test typecheck security ci clean +.PHONY: help install dev build lint format check test test-all security ci clean mypy-gate typecheck-gate help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' @@ -27,8 +27,14 @@ check: ## Full check: lint + format + typecheck + test mypy app/ --ignore-missing-imports pytest tests/ -x -q -m "not integration" -typecheck: ## MyPy type check - mypy app/ --ignore-missing-imports +typecheck: ## MyPy type check (full codebase, informational) + mypy app/ --config-file mypy.ini + +mypy-gate: ## MyPy gate for clean modules (authoritative) + mypy app/domains/auth/ app/core/redis.py --config-file mypy-gate.ini + +typecheck-gate: ## Alias for mypy-gate + $(MAKE) mypy-gate test: ## Run tests (non-integration) pytest tests/ -x -q -m "not integration" @@ -43,7 +49,7 @@ security: ## Security audit ci: ## Full CI pipeline ruff check app/ tests/ ruff format --check app/ tests/ - mypy app/ --ignore-missing-imports + $(MAKE) mypy-gate pytest tests/ -x -q -m "not integration" clean: ## Remove build artifacts diff --git a/app/mcp_router.py b/app/_archive/legacy_2026_07/mcp_router_2026_07.py similarity index 100% rename from app/mcp_router.py rename to app/_archive/legacy_2026_07/mcp_router_2026_07.py diff --git a/app/admin_backend.py b/app/admin_backend.py index dfb357e..5e5889d 100644 --- a/app/admin_backend.py +++ b/app/admin_backend.py @@ -1,1070 +1,19 @@ +"""Deprecated shim - re-exports app.domains.admin. + +Migrate imports from `app.admin_backend` to `app.domains.admin`. +This shim will be removed in Phase 3 cleanup. """ -RMI Admin Backend - Complete Administration System -=================================================== -A hardened, feature-rich admin panel for the RugMunch Intelligence Platform. - -Features: - • Role-Based Access Control (RBAC) - superadmin, admin, moderator, viewer - • Audit Logging - every action logged with IP, timestamp, before/after state - • Rate Limiting - per-endpoint, per-user, per-IP limits - • IP Blocking / Allowlisting - ban bad actors by IP - • 2FA Support - TOTP-based two-factor authentication - • Session Management - active sessions, force logout, expiry control - • System Health - real-time metrics, disk, memory, CPU, service status - • User Management - CRUD, ban/unban, role assignment, activity tracking - • Configuration Management - env vars, feature flags, system settings - • Security Dashboard - failed logins, blocked IPs, threat alerts - • Content Management - announcements, blog posts, SEO settings - • Financial Dashboard - x402 revenue, payment tracking, analytics - • API Key Management - rotate, revoke, scope-limited keys - • Webhook Management - configure, test, monitor webhooks - • Backup & Restore - database, config, snapshot management - -Security: - - All endpoints require admin authentication (JWT + role check) - - Audit log of every admin action (immutable, append-only) - - Rate limiting on all admin endpoints (stricter than public) - - IP allowlist for admin access (optional) - - Session timeout and concurrent session limits - - Password policy enforcement for admin accounts - - Automatic lockout after failed login attempts -""" - from __future__ import annotations -import hashlib -import json -import logging -import os -import secrets -import time -from dataclasses import asdict, dataclass -from datetime import datetime, timedelta -from enum import StrEnum -from typing import Any - -from fastapi import HTTPException, Request - -logger = logging.getLogger("rmi_admin_backend") - - -# ── Role Definitions ────────────────────────────────────────── - - -class AdminRole(StrEnum): - """Admin role hierarchy. Higher = more permissions.""" - - SUPERADMIN = "superadmin" # Full access, can manage other admins - ADMIN = "admin" # Full access except admin management - MODERATOR = "moderator" # Content + user management, no system config - VIEWER = "viewer" # Read-only access to dashboards - SUPPORT = "support" # User management only, read-only system - - -# Permission matrix: role -> set of allowed permissions -PERMISSIONS = { - AdminRole.SUPERADMIN: { - "*", # All permissions - }, - AdminRole.ADMIN: { - "dashboard.read", - "users.read", - "users.write", - "users.ban", - "content.read", - "content.write", - "system.read", - "system.write", - "security.read", - "security.write", - "financial.read", - "financial.write", - "api_keys.read", - "api_keys.write", - "webhooks.read", - "webhooks.write", - "backups.read", - "backups.write", - "settings.read", - "settings.write", - "logs.read", - "analytics.read", - "token_deploy.read", - "token_deploy.write", - }, - AdminRole.MODERATOR: { - "dashboard.read", - "users.read", - "users.write", - "users.ban", - "content.read", - "content.write", - "security.read", - "logs.read", - "analytics.read", - }, - AdminRole.VIEWER: { - "dashboard.read", - "users.read", - "content.read", - "system.read", - "security.read", - "financial.read", - "analytics.read", - "logs.read", - }, - AdminRole.SUPPORT: { - "dashboard.read", - "users.read", - "users.write", - "content.read", - "logs.read", - "analytics.read", - }, -} - - -def has_permission(role: AdminRole, permission: str) -> bool: - """Check if a role has a specific permission.""" - if role == AdminRole.SUPERADMIN: - return True - perms = PERMISSIONS.get(role, set()) - return permission in perms or "*" in perms - - -# ── Audit Log ─────────────────────────────────────────────────── - - -@dataclass -class AuditLogEntry: - """Single audit log entry.""" - - entry_id: str - timestamp: str - admin_id: str - admin_email: str - action: str # e.g., "user.ban", "token.deploy", "config.update" - resource_type: str # e.g., "user", "token", "config" - resource_id: str - ip_address: str - user_agent: str - before_state: dict | None = None - after_state: dict | None = None - status: str = "success" # success, failed, denied - reason: str = "" - session_id: str = "" - request_id: str = "" - - def to_dict(self) -> dict: - return asdict(self) - - -class AuditLogger: - """ - Immutable audit logging system. - Stores to Redis (time-series) + file backup + optional Supabase. - """ - - @staticmethod - async def log( - admin_id: str, - admin_email: str, - action: str, - resource_type: str, - resource_id: str, - ip_address: str = "", - user_agent: str = "", - before_state: dict | None = None, - after_state: dict | None = None, - status: str = "success", - reason: str = "", - session_id: str = "", - request_id: str = "", - ) -> bool: - """Log an admin action.""" - entry = AuditLogEntry( - entry_id=f"audit_{int(time.time() * 1000)}_{secrets.token_hex(4)}", - timestamp=datetime.utcnow().isoformat(), - admin_id=admin_id, - admin_email=admin_email, - action=action, - resource_type=resource_type, - resource_id=resource_id, - ip_address=ip_address, - user_agent=user_agent, - before_state=before_state, - after_state=after_state, - status=status, - reason=reason, - session_id=session_id, - request_id=request_id, - ) - - # Write to Redis (time-series list) - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Add to time-series list - key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" - await r.lpush(key, json.dumps(entry.to_dict())) - - # Keep only last 30 days in Redis - await r.ltrim(key, 0, 9999) - - # Add to admin-specific log - await r.lpush(f"audit_log:admin:{admin_id}", json.dumps(entry.to_dict())) - - # Add to action-specific index - await r.sadd(f"audit_log:actions:{action}", entry.entry_id) - - except Exception as e: - logger.error(f"Redis audit log failed: {e}") - - # Write to local file (append-only, immutable) - try: - log_file = f"/var/log/rmi/audit_{datetime.utcnow().strftime('%Y-%m')}.jsonl" - os.makedirs(os.path.dirname(log_file), exist_ok=True) - with open(log_file, "a") as f: - f.write(json.dumps(entry.to_dict()) + "\n") - except Exception as e: - logger.error(f"File audit log failed: {e}") - - # Write to Supabase if available - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("audit_logs").insert(entry.to_dict()).execute() - except Exception as e: - logger.error(f"Supabase audit log failed: {e}") - - return True - - @staticmethod - async def query( - admin_id: str | None = None, - action: str | None = None, - resource_type: str | None = None, - start_date: str | None = None, - end_date: str | None = None, - limit: int = 100, - offset: int = 0, - ) -> list[AuditLogEntry]: - """Query audit logs.""" - entries = [] - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Get today's log - key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" - logs = await r.lrange(key, offset, offset + limit - 1) - - for log in logs: - data = json.loads(log) - - # Filter - if admin_id and data.get("admin_id") != admin_id: - continue - if action and data.get("action") != action: - continue - if resource_type and data.get("resource_type") != resource_type: - continue - - entries.append(AuditLogEntry(**data)) - - except Exception as e: - logger.error(f"Audit query failed: {e}") - - return entries - - -# ── Security Manager ────────────────────────────────────────── - - -class SecurityManager: - """ - Security hardening for admin backend. - - Rate limiting - - IP blocking - - Failed login tracking - - Session management - """ - - # Rate limits: (max_requests, window_seconds) - RATE_LIMITS = { # noqa: RUF012 - "default": (60, 60), # 60/minute - "login": (5, 300), # 5/5min (strict for login) - "admin_write": (30, 60), # 30/minute for write ops - "token_deploy": (10, 3600), # 10/hour for token deploy - "snapshot": (20, 3600), # 20/hour for snapshots - "airdrop": (5, 3600), # 5/hour for airdrops - } - - @staticmethod - async def check_rate_limit( - identifier: str, - limit_type: str = "default", - ) -> dict[str, Any]: - """Check if request is within rate limit.""" - max_req, window = SecurityManager.RATE_LIMITS.get(limit_type, (60, 60)) - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - key = f"rate_limit:{limit_type}:{identifier}" - now = int(time.time()) - - # Clean old entries - await r.zremrangebyscore(key, 0, now - window) - - # Count current requests - count = await r.zcard(key) - - if count >= max_req: - # Get time until reset - oldest = await r.zrange(key, 0, 0, withscores=True) - reset_at = oldest[0][1] + window if oldest else now + window - - return { - "allowed": False, - "remaining": 0, - "reset_at": reset_at, - "limit": max_req, - "window": window, - } - - # Add current request - await r.zadd(key, {str(now): now}) - await r.expire(key, window) - - return { - "allowed": True, - "remaining": max_req - count - 1, - "reset_at": now + window, - "limit": max_req, - "window": window, - } - - except Exception as e: - logger.error(f"Rate limit check failed: {e}") - # Fail open (allow request) if Redis is down - return {"allowed": True, "remaining": -1, "reset_at": 0} - - @staticmethod - async def block_ip(ip: str, reason: str = "", duration_hours: int = 24) -> bool: - """Block an IP address.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - key = f"blocked_ip:{ip}" - data = { - "ip": ip, - "reason": reason, - "blocked_at": datetime.utcnow().isoformat(), - "expires_at": (datetime.utcnow() + timedelta(hours=duration_hours)).isoformat(), - "duration_hours": duration_hours, - } - - await r.setex(key, duration_hours * 3600, json.dumps(data)) - await r.sadd("blocked_ips:all", ip) - - logger.warning(f"IP blocked: {ip} for {duration_hours}h - {reason}") - return True - - except Exception as e: - logger.error(f"IP block failed: {e}") - return False - - @staticmethod - async def unblock_ip(ip: str) -> bool: - """Unblock an IP address.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - await r.delete(f"blocked_ip:{ip}") - await r.srem("blocked_ips:all", ip) - - return True - - except Exception as e: - logger.error(f"IP unblock failed: {e}") - return False - - @staticmethod - async def is_ip_blocked(ip: str) -> dict | None: - """Check if IP is blocked. Returns block info or None.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - data = await r.get(f"blocked_ip:{ip}") - if data: - return json.loads(data) - - return None - - except Exception as e: - logger.error(f"IP block check failed: {e}") - return None - - @staticmethod - async def track_failed_login(identifier: str) -> int: - """Track failed login attempts. Returns current count.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - key = f"failed_logins:{identifier}" - count = await r.incr(key) - await r.expire(key, 3600) # 1 hour window - - # Auto-block after 5 failed attempts - if count >= 5: - ip = identifier.split(":")[-1] if ":" in identifier else identifier - await SecurityManager.block_ip(ip, "Too many failed login attempts", 1) - - return count - - except Exception as e: - logger.error(f"Failed login tracking error: {e}") - return 0 - - @staticmethod - async def reset_failed_login(identifier: str) -> bool: - """Reset failed login counter (on successful login).""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - await r.delete(f"failed_logins:{identifier}") - return True - - except Exception as e: - logger.error(f"Reset failed login error: {e}") - return False - - -# ── Session Manager ─────────────────────────────────────────── - - -class SessionManager: - """ - Admin session management. - - Track active sessions - - Force logout - - Concurrent session limits - - Session expiry - """ - - MAX_CONCURRENT_SESSIONS = 3 - SESSION_TIMEOUT_HOURS = 8 - - @staticmethod - async def create_session( - admin_id: str, - admin_email: str, - role: AdminRole, - ip_address: str, - user_agent: str, - ) -> str: - """Create a new admin session.""" - session_id = f"sess_{secrets.token_urlsafe(16)}" - - session_data = { - "session_id": session_id, - "admin_id": admin_id, - "admin_email": admin_email, - "role": role.value if isinstance(role, AdminRole) else role, - "ip_address": ip_address, - "user_agent": user_agent, - "created_at": datetime.utcnow().isoformat(), - "expires_at": (datetime.utcnow() + timedelta(hours=SessionManager.SESSION_TIMEOUT_HOURS)).isoformat(), - "last_active": datetime.utcnow().isoformat(), - "is_active": True, - } - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Store session - key = f"admin_session:{session_id}" - await r.setex(key, SessionManager.SESSION_TIMEOUT_HOURS * 3600, json.dumps(session_data)) - - # Add to admin's session list - await r.sadd(f"admin_sessions:{admin_id}", session_id) - - # Enforce max concurrent sessions - sessions = await r.smembers(f"admin_sessions:{admin_id}") - if len(sessions) > SessionManager.MAX_CONCURRENT_SESSIONS: - # Remove oldest sessions - sorted_sessions = sorted(sessions) - for old_session in sorted_sessions[: -SessionManager.MAX_CONCURRENT_SESSIONS]: - await SessionManager.destroy_session(old_session) - - return session_id - - except Exception as e: - logger.error(f"Session creation failed: {e}") - return "" - - @staticmethod - async def validate_session(session_id: str) -> dict | None: - """Validate and refresh a session.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - data = await r.get(f"admin_session:{session_id}") - if not data: - return None - - session = json.loads(data) - - # Check expiry - expires = datetime.fromisoformat(session["expires_at"]) - if datetime.utcnow() > expires: - await SessionManager.destroy_session(session_id) - return None - - # Update last active - session["last_active"] = datetime.utcnow().isoformat() - await r.setex( - f"admin_session:{session_id}", - SessionManager.SESSION_TIMEOUT_HOURS * 3600, - json.dumps(session), - ) - - return session - - except Exception as e: - logger.error(f"Session validation failed: {e}") - return None - - @staticmethod - async def destroy_session(session_id: str) -> bool: - """Destroy a session (logout).""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Get session data for admin_id - data = await r.get(f"admin_session:{session_id}") - if data: - session = json.loads(data) - admin_id = session.get("admin_id") - if admin_id: - await r.srem(f"admin_sessions:{admin_id}", session_id) - - await r.delete(f"admin_session:{session_id}") - return True - - except Exception as e: - logger.error(f"Session destroy failed: {e}") - return False - - @staticmethod - async def get_active_sessions(admin_id: str) -> list[dict]: - """Get all active sessions for an admin.""" - sessions = [] - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - session_ids = await r.smembers(f"admin_sessions:{admin_id}") - for sid in session_ids: - data = await r.get(f"admin_session:{sid}") - if data: - session = json.loads(data) - # Check if still valid - expires = datetime.fromisoformat(session["expires_at"]) - if datetime.utcnow() < expires: - sessions.append(session) - else: - await r.srem(f"admin_sessions:{admin_id}", sid) - - except Exception as e: - logger.error(f"Get active sessions failed: {e}") - - return sessions - - @staticmethod - async def destroy_all_sessions(admin_id: str) -> int: - """Destroy all sessions for an admin (force logout everywhere).""" - count = 0 - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - session_ids = await r.smembers(f"admin_sessions:{admin_id}") - for sid in session_ids: - await r.delete(f"admin_session:{sid}") - count += 1 - - await r.delete(f"admin_sessions:{admin_id}") - - except Exception as e: - logger.error(f"Destroy all sessions failed: {e}") - - return count - - -# ── Admin User Store ────────────────────────────────────────── - - -class AdminUserStore: - """ - Store and manage admin user accounts. - Uses Redis as primary + Supabase as backup. - """ - - @staticmethod - async def get_admin(admin_id: str) -> dict | None: - """Get admin by ID.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - data = await r.hget("rmi:admins", admin_id) - if data: - return json.loads(data) - - return None - - except Exception as e: - logger.error(f"Get admin failed: {e}") - return None - - @staticmethod - async def get_admin_by_email(email: str) -> dict | None: - """Get admin by email.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - admin_id = await r.hget("rmi:admins:email", email.lower()) - if admin_id: - return await AdminUserStore.get_admin(admin_id) - - return None - - except Exception as e: - logger.error(f"Get admin by email failed: {e}") - return None - - @staticmethod - async def save_admin(admin: dict) -> bool: - """Save admin user.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - await r.hset("rmi:admins", admin["id"], json.dumps(admin)) - await r.hset("rmi:admins:email", admin["email"].lower(), admin["id"]) - - # Also save to Supabase - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("admin_users").upsert(admin).execute() - except Exception: - pass - - return True - - except Exception as e: - logger.error(f"Save admin failed: {e}") - return False - - @staticmethod - async def list_admins() -> list[dict]: - """List all admin users.""" - admins = [] - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - all_admins = await r.hgetall("rmi:admins") - for data in all_admins.values(): - admins.append(json.loads(data)) - - except Exception as e: - logger.error(f"List admins failed: {e}") - - return admins - - @staticmethod - async def create_admin( - email: str, - password: str, - role: AdminRole = AdminRole.VIEWER, - created_by: str = "", - ) -> dict | None: - """Create a new admin user.""" - from app.domains.auth.passwords import hash_password - - # Check if email already exists - existing = await AdminUserStore.get_admin_by_email(email) - if existing: - return None - - admin_id = hashlib.sha256(email.lower().encode()).hexdigest()[:16] - - admin = { - "id": admin_id, - "email": email, - "password_hash": hash_password(password), - "role": role.value, - "is_active": True, - "created_at": datetime.utcnow().isoformat(), - "created_by": created_by, - "last_login": None, - "login_count": 0, - "two_factor_enabled": False, - "two_factor_secret": None, - "ip_allowlist": [], - "metadata": {}, - } - - await AdminUserStore.save_admin(admin) - - # Remove password hash from returned dict - safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} - return safe_admin - - @staticmethod - async def verify_admin_login(email: str, password: str) -> dict | None: - """Verify admin login credentials.""" - from app.domains.auth.passwords import verify_password - - admin = await AdminUserStore.get_admin_by_email(email) - if not admin: - return None - - if not admin.get("is_active", True): - return None - - if not verify_password(password, admin.get("password_hash", "")): - return None - - # Update last login - admin["last_login"] = datetime.utcnow().isoformat() - admin["login_count"] = admin.get("login_count", 0) + 1 - await AdminUserStore.save_admin(admin) - - # Remove sensitive data - safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} - return safe_admin - - -# ── System Health Monitor ───────────────────────────────────── - - -class SystemHealthMonitor: - """ - Monitor system health and performance. - """ - - @staticmethod - async def get_system_health() -> dict[str, Any]: - """Get comprehensive system health.""" - import psutil - - # CPU - cpu_percent = psutil.cpu_percent(interval=0.5) - cpu_count = psutil.cpu_count() - - # Memory - memory = psutil.virtual_memory() - - # Disk - disk = psutil.disk_usage("/") - - # Network - net_io = psutil.net_io_counters() - - # Process info - backend_pid = os.getpid() - backend_proc = psutil.Process(backend_pid) - - # Check services - services = { - "redis": await SystemHealthMonitor._check_redis(), - "supabase": await SystemHealthMonitor._check_supabase(), - "backend": {"status": "running", "pid": backend_pid}, - } - - return { - "timestamp": datetime.utcnow().isoformat(), - "cpu": { - "percent": cpu_percent, - "count": cpu_count, - "per_cpu": psutil.cpu_percent(percpu=True, interval=0.1), - }, - "memory": { - "total_gb": round(memory.total / (1024**3), 2), - "available_gb": round(memory.available / (1024**3), 2), - "percent": memory.percent, - "used_gb": round(memory.used / (1024**3), 2), - }, - "disk": { - "total_gb": round(disk.total / (1024**3), 2), - "used_gb": round(disk.used / (1024**3), 2), - "free_gb": round(disk.free / (1024**3), 2), - "percent": round(disk.used / disk.total * 100, 1), - }, - "network": { - "bytes_sent_mb": round(net_io.bytes_sent / (1024**2), 2), - "bytes_recv_mb": round(net_io.bytes_recv / (1024**2), 2), - }, - "backend": { - "pid": backend_pid, - "memory_mb": round(backend_proc.memory_info().rss / (1024**2), 2), - "cpu_percent": backend_proc.cpu_percent(interval=0.2), - "threads": backend_proc.num_threads(), - "open_files": len(backend_proc.open_files()), - "connections": len(backend_proc.connections()), - }, - "services": services, - "uptime_seconds": int(time.time() - psutil.boot_time()), - } - - @staticmethod - async def _check_redis() -> dict: - """Check Redis connection.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - await r.ping() - info = await r.info() - return { - "status": "connected", - "version": info.get("redis_version", "unknown"), - "used_memory_mb": round(info.get("used_memory", 0) / (1024**2), 2), - "connected_clients": info.get("connected_clients", 0), - "total_keys": info.get("db0", {}).get("keys", 0) if isinstance(info.get("db0"), dict) else 0, - } - except Exception as e: - return {"status": "error", "error": str(e)} - - @staticmethod - async def _check_supabase() -> dict: - """Check Supabase connection.""" - try: - supabase_url = os.getenv("SUPABASE_URL") - if not supabase_url: - return {"status": "not_configured"} - - import httpx - - async with httpx.AsyncClient() as client: - r = await client.get(f"{supabase_url}/rest/v1/", timeout=5) - return { - "status": "connected" if r.status_code < 500 else "error", - "url": supabase_url, - "response_code": r.status_code, - } - except Exception as e: - return {"status": "error", "error": str(e)} - - -# ── Convenience: Admin Auth Decorator ───────────────────────── - - -async def require_admin( - request: Request, - required_permission: str = "", - min_role: AdminRole = AdminRole.VIEWER, -) -> dict[str, Any]: - """ - Verify admin authentication and authorization. - - Usage: - admin = await require_admin(request, "users.write", AdminRole.ADMIN) - """ - # Get session token from header - session_token = request.headers.get("X-Admin-Session", "") - if not session_token: - raise HTTPException(status_code=401, detail="Admin session required") - - # Validate session - session = await SessionManager.validate_session(session_token) - if not session: - raise HTTPException(status_code=401, detail="Invalid or expired session") - - # Check IP block - client_ip = request.client.host if request.client else "" - block_info = await SecurityManager.is_ip_blocked(client_ip) - if block_info: - raise HTTPException(status_code=403, detail="IP blocked") - - # Get admin - admin = await AdminUserStore.get_admin(session["admin_id"]) - if not admin or not admin.get("is_active", True): - raise HTTPException(status_code=403, detail="Admin account inactive") - - # Check role level - role = AdminRole(admin.get("role", "viewer")) - role_order = { - AdminRole.VIEWER: 0, - AdminRole.SUPPORT: 1, - AdminRole.MODERATOR: 2, - AdminRole.ADMIN: 3, - AdminRole.SUPERADMIN: 4, - } - if role_order[role] < role_order[min_role]: - raise HTTPException(status_code=403, detail="Insufficient privileges") - - # Check specific permission - if required_permission and not has_permission(role, required_permission): - raise HTTPException(status_code=403, detail="Permission denied") - - # Check rate limit - rate_check = await SecurityManager.check_rate_limit( - f"admin:{admin['id']}", - "admin_write" if required_permission.endswith(".write") else "default", - ) - if not rate_check["allowed"]: - raise HTTPException(status_code=429, detail=f"Rate limit exceeded. Reset at {rate_check['reset_at']}") - - # Log access - await AuditLogger.log( - admin_id=admin["id"], - admin_email=admin["email"], - action="admin.access", - resource_type="endpoint", - resource_id=str(request.url.path), - ip_address=client_ip, - user_agent=request.headers.get("user-agent", ""), - session_id=session_token, - ) - - return { - "admin": admin, - "session": session, - "role": role, - } +from app.domains.admin import ( # noqa: F401 + PERMISSIONS, + AdminRole, + AdminUserStore, + AuditLogEntry, + AuditLogger, + SecurityManager, + SessionManager, + SystemHealthMonitor, + has_permission, + require_admin, +) diff --git a/app/alert_pipeline.py b/app/alert_pipeline.py index f93c7f9..9b4d5a4 100644 --- a/app/alert_pipeline.py +++ b/app/alert_pipeline.py @@ -1,370 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.alert_pipeline. + +Migrate imports from `app.alert_pipeline` to `app.domains.intelligence.alert_pipeline`. +This shim will be removed in Phase 3 cleanup. """ -RMI Alert Pipeline - Real-time threat intelligence from scanners. -================================================================= -Feeds the Live Intel panel, WebSocket streams, and alert endpoints. +from __future__ import annotations -Data sources: - - SENTINEL scanner (risk scores, scam detection) - - GoPlus security API (honeypot, tax, proxy checks) - - DexScreener new pairs (fresh launches to scan) - - Our own RAG scam patterns - - Wallet label cross-references - -Alert flow: - Scanner → alert_pipeline.push_alert() → Redis sorted set + pub/sub - Homepage reads from /api/v1/alerts/recent - Sidebar reads from /api/v1/alerts/count - WebSocket streams to connected clients -""" - -import json -import logging -import os -import time -from datetime import UTC, datetime - -logger = logging.getLogger("alert_pipeline") - -REDIS_HOST = os.getenv("REDIS_HOST", "rmi-redis") -REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) -REDIS_PW = os.getenv("REDIS_PASSWORD", "") - -ALERT_KEY = "rmi:alerts:recent" -ALERT_COUNT_KEY = "rmi:alerts:count:active" -ALERT_MAX = 500 # Keep last 500 alerts - - -async def _get_redis(): - """Get async Redis connection.""" - import redis.asyncio as aioredis - - return aioredis.Redis( - host=REDIS_HOST, - port=REDIS_PORT, - password=REDIS_PW or None, - decode_responses=True, - ) - - -async def get_active_alert_count() -> int: - """Get count of active (unacknowledged) alerts.""" - try: - r = await _get_redis() - count = await r.zcard(ALERT_KEY) - await r.close() - return count - except Exception as e: - logger.debug(f"Alert count error: {e}") - return 0 - - -async def push_alert( - alert_type: str, - title: str, - description: str = "", - severity: str = "high", - chain: str = "unknown", - token: str = "", - token_symbol: str = "", - wallet: str = "", - risk_score: int = 0, - metadata: dict | None = None, -) -> str: - """ - Push a new alert to the pipeline. - Returns alert_id. - """ - alert_id = f"alt_{int(time.time())}_{os.urandom(4).hex()}" - - alert = { - "id": alert_id, - "type": alert_type, - "title": title, - "description": description, - "severity": severity, - "chain": chain, - "token": token, - "token_symbol": token_symbol, - "wallet": wallet, - "risk_score": risk_score, - "acknowledged": False, - "created_at": datetime.now(UTC).isoformat(), - **(metadata or {}), - } - - try: - r = await _get_redis() - score = time.time() - await r.zadd(ALERT_KEY, {json.dumps(alert): score}) - # Trim old alerts - await r.zremrangebyrank(ALERT_KEY, 0, -(ALERT_MAX + 1)) - # Pub/sub for WebSocket streaming - await r.publish("rmi:ws:alerts", json.dumps(alert)) - await r.close() - logger.info(f"Alert pushed: {alert_type} | {title[:60]}") - except Exception as e: - logger.error(f"Failed to push alert: {e}") - - return alert_id - - -async def get_recent_alerts(limit: int = 20, severity: str = "") -> list[dict]: - """Get recent alerts with optional severity filter. Normalizes old/new formats.""" - try: - r = await _get_redis() - raw = await r.zrevrange(ALERT_KEY, 0, limit * 2 - 1) - await r.close() - - alerts = [] - for entry in raw: - try: - alert = json.loads(entry) - - # Normalize old alert format to new - if "title" not in alert: - alert["title"] = alert.get("message", alert.get("event", "Unknown alert")) - if "description" not in alert: - desc_parts = [] - if alert.get("symbol"): - desc_parts.append(f"Token: {alert['symbol']}") - if alert.get("risk_score"): - desc_parts.append(f"Risk: {alert['risk_score']}/100") - flags = alert.get("risk_flags", []) - if flags: - desc_parts.append("; ".join(str(f)[:80] for f in flags[:2])) - alert["description"] = " | ".join(desc_parts) - if "severity" not in alert: - score = alert.get("risk_score", 50) - alert["severity"] = "critical" if score >= 85 else "high" if score >= 65 else "medium" - if "chain" not in alert: - alert["chain"] = alert.get("chain", "unknown") - - if severity and alert.get("severity") != severity: - continue - alerts.append(alert) - if len(alerts) >= limit: - break - except json.JSONDecodeError: - pass - - return alerts - except Exception as e: - logger.error(f"get_recent_alerts error: {e}") - return [] - - -# ── Alert generators - called by cron jobs or on-demand ────────── - - -async def scan_solana_new_pairs(limit: int = 5) -> int: - """ - Scan latest Solana pairs from DexScreener for scam patterns. - Pushes alerts for high-risk tokens. - Returns number of alerts generated. - """ - import httpx - - pushed = 0 - - try: - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.get("https://api.dexscreener.com/latest/dex/search", params={"q": "SOL USDC"}) - if resp.status_code != 200: - return 0 - - data = resp.json() - pairs = data.get("pairs", [])[:limit] - - for pair in pairs: - token_addr = pair.get("baseToken", {}).get("address", "") - token_name = pair.get("baseToken", {}).get("name", "Unknown") - token_symbol = pair.get("baseToken", {}).get("symbol", "???") - liquidity = float(pair.get("liquidity", {}).get("usd", 0) or 0) - volume = float(pair.get("volume", {}).get("h24", 0) or 0) - price_change = float(pair.get("priceChange", {}).get("h24", 0) or 0) - created_at = pair.get("pairCreatedAt", 0) - age_hours = (time.time() - created_at / 1000) / 3600 if created_at else 999 - - # Risk heuristics - risk_flags = [] - - if liquidity < 1000: - risk_flags.append("low_liquidity") - if volume == 0: - risk_flags.append("no_volume") - if price_change < -80: - risk_flags.append(f"crashed_{abs(price_change):.0f}%") - if age_hours < 1 and liquidity < 5000: - risk_flags.append("fresh_launch_low_liq") - if price_change > 500: - risk_flags.append(f"pumped_{price_change:.0f}%") - - if risk_flags: - await push_alert( - alert_type="new_pair_risk", - title=f"{token_symbol}: {' | '.join(risk_flags[:2])}", - description=f"New pair {token_name} ({token_symbol}) on Solana. " - f"Liquidity: ${liquidity:,.0f}, Age: {age_hours:.1f}h, " - f"24h change: {price_change:+.1f}%", - severity="critical" if len(risk_flags) >= 3 else "high", - chain="solana", - token=token_addr, - token_symbol=token_symbol, - risk_score=min(90, len(risk_flags) * 25), - metadata={ - "risk_flags": risk_flags, - "liquidity_usd": liquidity, - "age_hours": age_hours, - }, - ) - pushed += 1 - - return pushed - except Exception as e: - logger.warning(f"Solana scan error: {e}") - return pushed - - -async def scan_known_scams(limit: int = 3) -> int: - """ - Check our RAG known_scams collection for recently added entries. - Pushes alerts for new scam patterns detected. - """ - pushed = 0 - try: - r = await _get_redis() - # Check for recent scam pattern additions - scam_ids = await r.smembers("rag:idx:known_scams") - - recent_count = 0 - for sid in list(scam_ids)[:20]: - doc = await r.get(f"rag:known_scams:{sid}") - if doc: - try: - data = json.loads(doc) - added = data.get("metadata", {}).get("added_at", "") - if added: - age_h = (time.time() - datetime.fromisoformat(added.replace("Z", "+00:00")).timestamp()) / 3600 - if age_h < 24: - recent_count += 1 - except Exception: - pass - - await r.close() - - if recent_count > 0: - await push_alert( - alert_type="scam_pattern_update", - title=f"{recent_count} new scam patterns detected in last 24h", - description="New rug pull, honeypot, or phishing patterns added to the knowledge base.", - severity="high", - chain="multi", - risk_score=85, - ) - pushed += 1 - - return pushed - except Exception as e: - logger.warning(f"Known scams scan error: {e}") - return pushed - - -async def run_alert_scan() -> dict[str, int]: - """ - Run a full alert scan across all sources. - Called by cron job every 15 minutes. - """ - results = {} - - # Scan Solana new pairs - results["solana_pairs"] = await scan_solana_new_pairs(limit=8) - - # Scan known scams - results["known_scams"] = await scan_known_scams() - - total = sum(results.values()) - logger.info(f"Alert scan complete: {total} new alerts ({results})") - return results - - -# ── Seed some initial alerts if Redis is empty ──────────────────── - - -async def seed_initial_alerts(): - """Seed baseline alerts so the system isn't empty on first run.""" - r = await _get_redis() - existing = await r.zcard(ALERT_KEY) - await r.close() - - if existing > 0: - return # Already has alerts - - seeds = [ - ( - "honeypot", - "Honeypot detected on Base: 0xdead...", - "Token has sell restrictions and blacklist. Buyers cannot exit.", - "critical", - "base", - ), - ( - "whale", - "Whale moved 5M USDC to Binance", - "Wallet 0xABCD... transferred $5M USDC to Binance hot wallet. Possible sell pressure.", - "high", - "ethereum", - ), - ( - "rugpull", - "Liquidity removed from $SCAM token", - "100% of liquidity pool withdrawn by deployer. Token is now worthless.", - "critical", - "solana", - ), - ( - "bundler", - "Sniper bundle detected: $NEWLAUNCH", - "Coordinated wallet cluster bought 60% of supply in first block.", - "high", - "solana", - ), - ( - "contract", - "Unverified proxy contract found", - "Token uses upgradeable proxy with unverified implementation. Owner can change logic.", - "high", - "base", - ), - ( - "concentration", - "90% supply held by 3 wallets on $DANGER", - "Extreme holder concentration. Classic rug pull setup.", - "critical", - "ethereum", - ), - ( - "wash_trade", - "Wash trading detected on $FAKEVOL", - "95% of volume is self-trading between linked wallets.", - "high", - "bsc", - ), - ( - "phishing", - "Fake airdrop targeting $BONK holders", - "Phishing site detected posing as official BONK airdrop. Users losing funds.", - "critical", - "solana", - ), - ] - - for alert_type, title, desc, severity, chain in seeds: - await push_alert( - alert_type=alert_type, - title=title, - description=desc, - severity=severity, - chain=chain, - ) - - logger.info(f"Seeded {len(seeds)} initial alerts") +from app.domains.intelligence.alert_pipeline import * # noqa: F403 diff --git a/app/api/v1/mcp/__init__.py b/app/api/v1/mcp/__init__.py index 036000c..9ba91a7 100644 --- a/app/api/v1/mcp/__init__.py +++ b/app/api/v1/mcp/__init__.py @@ -1,4 +1,10 @@ -"""MCP v1 routes.""" -from .router import router +"""Deprecated shim - MCP router moved to app.domains.mcp.router. + +Migrate imports from `app.api.v1.mcp` to `app.domains.mcp`. +This shim will be removed in Phase 3 cleanup. +""" +from __future__ import annotations + +from app.domains.mcp.router import router __all__ = ["router"] diff --git a/app/bulletin_board.py b/app/bulletin_board.py index 724a761..17641a3 100644 --- a/app/bulletin_board.py +++ b/app/bulletin_board.py @@ -1,903 +1,21 @@ +"""Deprecated shim - re-exports app.domains.bulletin. + +Migrate imports from `app.bulletin_board` to `app.domains.bulletin`. +This shim will be removed in Phase 3 cleanup. """ -RMI Bulletin Board Management System -===================================== -A full content management backend for announcements, news, alerts, platform -communications, and community bulletin boards. - -Features: - • Posts - CRUD with rich text, attachments, scheduling, expiry - • Categories - organize by type (news, alert, update, promo, system) - • Targeting - audience segmentation (all, free, premium, admins, specific tiers) - • Moderation - draft/review/published/archived workflow, approval chains - • Pinning - sticky posts, priority ordering - • Analytics - views, clicks, engagement tracking per post - • Comments - threaded discussions on posts (optional) - • Notifications - push/email/Telegram alerts for critical posts - • Scheduling - publish at future date, auto-archive after expiry - • Versioning - track edit history, rollback capability - • Search - full-text search across all posts - • SEO - slug generation, meta tags, OpenGraph - -Security: - - All write operations require admin auth + content.write permission - - Audit log of every content change - - Rate limiting on publish operations - - Content sanitization (strip XSS, validate HTML) -""" - from __future__ import annotations -import html -import json -import logging -import os -import re -import time -from dataclasses import asdict, dataclass, field -from datetime import datetime -from enum import StrEnum -from typing import Any, ClassVar - -logger = logging.getLogger("rmi_bulletin_board") - - -# ── Enums ───────────────────────────────────────────────────── - - -class PostStatus(StrEnum): - DRAFT = "draft" - REVIEW = "review" - PUBLISHED = "published" - ARCHIVED = "archived" - SCHEDULED = "scheduled" - - -class PostCategory(StrEnum): - NEWS = "news" # Platform news - ALERT = "alert" # Security/urgent alerts - UPDATE = "update" # Feature updates - PROMO = "promo" # Promotions/offers - SYSTEM = "system" # System maintenance - COMMUNITY = "community" # Community posts - ANNOUNCEMENT = "announcement" # General announcements - TUTORIAL = "tutorial" # Guides/how-tos - - -class TargetAudience(StrEnum): - ALL = "all" - FREE = "free" - PREMIUM = "premium" - PRO = "pro" - ENTERPRISE = "enterprise" - ADMINS = "admins" - MODERATORS = "moderators" - - -class Priority(StrEnum): - LOW = "low" - NORMAL = "normal" - HIGH = "high" - CRITICAL = "critical" - - -# ── Data Models ───────────────────────────────────────────── - - -@dataclass -class Post: - """Bulletin board post.""" - - post_id: str - title: str - slug: str - content: str - summary: str - category: str - status: str - priority: str - target_audience: str - author_id: str - author_email: str - author_name: str - created_at: str - updated_at: str - published_at: str | None = None - scheduled_at: str | None = None - expires_at: str | None = None - archived_at: str | None = None - pinned: bool = False - pin_order: int = 0 - featured_image: str = "" - attachments: list[dict] = field(default_factory=list) - tags: list[str] = field(default_factory=list) - meta_title: str = "" - meta_description: str = "" - og_image: str = "" - view_count: int = 0 - click_count: int = 0 - engagement_score: float = 0.0 - version: int = 1 - edit_history: list[dict] = field(default_factory=list) - approved_by: str = "" - approved_at: str | None = None - notification_sent: bool = False - allow_comments: bool = False - comments_count: int = 0 - - def to_dict(self) -> dict: - return asdict(self) - - def to_public_dict(self) -> dict: - """Return public-safe version (no internal fields).""" - return { - "post_id": self.post_id, - "title": self.title, - "slug": self.slug, - "content": self.content, - "summary": self.summary, - "category": self.category, - "priority": self.priority, - "author_name": self.author_name, - "created_at": self.created_at, - "published_at": self.published_at, - "expires_at": self.expires_at, - "pinned": self.pinned, - "pin_order": self.pin_order, - "featured_image": self.featured_image, - "attachments": self.attachments, - "tags": self.tags, - "meta_title": self.meta_title, - "meta_description": self.meta_description, - "og_image": self.og_image, - "view_count": self.view_count, - "allow_comments": self.allow_comments, - "comments_count": self.comments_count, - } - - -@dataclass -class Comment: - """Comment on a post.""" - - comment_id: str - post_id: str - author_id: str - author_name: str - author_email: str - content: str - created_at: str - updated_at: str - parent_id: str | None = None - status: str = "approved" # approved, pending, rejected - likes: int = 0 - replies: list[dict] = field(default_factory=list) - - def to_dict(self) -> dict: - return asdict(self) - - -# ── Content Sanitizer ─────────────────────────────────────── - - -class ContentSanitizer: - """Sanitize user-generated content to prevent XSS.""" - - ALLOWED_TAGS: ClassVar[dict] ={ - "p", - "br", - "strong", - "b", - "em", - "i", - "u", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "ul", - "ol", - "li", - "a", - "img", - "blockquote", - "code", - "pre", - "table", - "thead", - "tbody", - "tr", - "td", - "th", - "div", - "span", - "hr", - "sub", - "sup", - "del", - "ins", - } - - ALLOWED_ATTRS: ClassVar[dict] ={ - "a": ["href", "title", "target"], - "img": ["src", "alt", "title", "width", "height"], - "div": ["class"], - "span": ["class"], - "code": ["class"], - "pre": ["class"], - } - - @staticmethod - def sanitize(text: str) -> str: - """Basic HTML sanitization.""" - if not text: - return "" - - # Escape HTML entities first - text = html.escape(text) - - # Then selectively un-escape allowed tags - # This is a simplified approach - in production use bleach or similar - # For now, strip all HTML tags for safety - text = re.sub(r"<[^>]+>", "", text) - - return text.strip() - - @staticmethod - def generate_slug(title: str) -> str: - """Generate URL-friendly slug from title.""" - slug = re.sub(r"[^\w\s-]", "", title.lower()) - slug = re.sub(r"[-\s]+", "-", slug) - return slug[:80] - - @staticmethod - def generate_summary(content: str, max_length: int = 200) -> str: - """Generate summary from content.""" - # Strip HTML - text = re.sub(r"<[^>]+>", "", content) - text = text.replace("\n", " ").strip() - if len(text) > max_length: - text = text[:max_length].rsplit(" ", 1)[0] + "..." - return text - - -# ── Bulletin Board Manager ──────────────────────────────────── - - -class BulletinBoardManager: - """ - Core manager for bulletin board operations. - Uses Redis as primary store + Supabase backup. - """ - - @staticmethod - async def _get_redis(): - import redis.asyncio as redis_lib - - return redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - @staticmethod - async def create_post( - title: str, - content: str, - category: str, - author_id: str, - author_email: str, - author_name: str = "", - priority: str = "normal", - target_audience: str = "all", - status: str = "draft", - featured_image: str = "", - attachments: list[dict] | None = None, - tags: list[str] | None = None, - scheduled_at: str | None = None, - expires_at: str | None = None, - pinned: bool = False, - allow_comments: bool = False, - meta_title: str = "", - meta_description: str = "", - ) -> Post: - """Create a new post.""" - post_id = f"post_{int(time.time())}_{os.urandom(4).hex()}" - slug = ContentSanitizer.generate_slug(title) - summary = ContentSanitizer.generate_summary(content) - now = datetime.utcnow().isoformat() - - # Sanitize content - content = ContentSanitizer.sanitize(content) - - post = Post( - post_id=post_id, - title=title[:200], - slug=slug, - content=content, - summary=summary, - category=category, - status=status, - priority=priority, - target_audience=target_audience, - author_id=author_id, - author_email=author_email, - author_name=author_name or author_email.split("@")[0], - created_at=now, - updated_at=now, - scheduled_at=scheduled_at, - expires_at=expires_at, - pinned=pinned, - featured_image=featured_image, - attachments=attachments or [], - tags=tags or [], - meta_title=meta_title or title[:70], - meta_description=meta_description or summary[:160], - allow_comments=allow_comments, - ) - - r = await BulletinBoardManager._get_redis() - - # Save post - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post.to_dict())) - - # Add to category index - await r.sadd(f"bulletin:category:{category}", post_id) - - # Add to status index - await r.sadd(f"bulletin:status:{status}", post_id) - - # Add to author index - await r.sadd(f"bulletin:author:{author_id}", post_id) - - # Add to audience index - await r.sadd(f"bulletin:audience:{target_audience}", post_id) - - # Add to pinned index if pinned - if pinned: - await r.sadd("bulletin:pinned", post_id) - await r.zadd("bulletin:pinned_order", {post_id: post.pin_order}) - - # Add to scheduled index if scheduled - if scheduled_at and status == "scheduled": - ts = int(datetime.fromisoformat(scheduled_at).timestamp()) - await r.zadd("bulletin:scheduled", {post_id: ts}) - - # Add to search index (simple word index) - words = set(re.findall(r"\w+", title.lower() + " " + content.lower())) - for word in words: - if len(word) > 2: - await r.sadd(f"bulletin:search:{word}", post_id) - - # Save to Supabase - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("bulletin_posts").insert(post.to_dict()).execute() - except Exception as e: - logger.error(f"Supabase bulletin post save failed: {e}") - - return post - - @staticmethod - async def get_post(post_id: str, increment_views: bool = False) -> Post | None: - """Get a post by ID.""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return None - - post_dict = json.loads(data) - - if increment_views and post_dict.get("status") == "published": - post_dict["view_count"] = post_dict.get("view_count", 0) + 1 - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - - return Post(**post_dict) - - @staticmethod - async def get_post_by_slug(slug: str) -> Post | None: - """Get a post by slug.""" - r = await BulletinBoardManager._get_redis() - # Search all posts for matching slug - all_posts = await r.hgetall("rmi:bulletin_posts") - for _post_id, data in all_posts.items(): - post_dict = json.loads(data) - if post_dict.get("slug") == slug: - return Post(**post_dict) - return None - - @staticmethod - async def update_post( - post_id: str, - updates: dict[str, Any], - editor_id: str = "", - editor_email: str = "", - ) -> Post | None: - """Update a post with versioning.""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return None - - post_dict = json.loads(data) - before_state = {k: post_dict.get(k) for k in updates if k in post_dict} - - # Record edit history - edit_entry = { - "edited_at": datetime.utcnow().isoformat(), - "edited_by": editor_id, - "editor_email": editor_email, - "changes": before_state, - "version": post_dict.get("version", 1), - } - - history = post_dict.get("edit_history", []) - history.append(edit_entry) - post_dict["edit_history"] = history - post_dict["version"] = post_dict.get("version", 1) + 1 - post_dict["updated_at"] = datetime.utcnow().isoformat() - - # Apply updates - for key, value in updates.items(): - if key == "content": - value = ContentSanitizer.sanitize(value) - post_dict["summary"] = ContentSanitizer.generate_summary(value) - if key == "title": - post_dict["slug"] = ContentSanitizer.generate_slug(value) - post_dict[key] = value - - # Handle status transitions - old_status = before_state.get("status") - new_status = post_dict.get("status") - if old_status != new_status: - # Update status indexes - if old_status: - await r.srem(f"bulletin:status:{old_status}", post_id) - await r.sadd(f"bulletin:status:{new_status}", post_id) - - if new_status == "published": - post_dict["published_at"] = datetime.utcnow().isoformat() - elif new_status == "archived": - post_dict["archived_at"] = datetime.utcnow().isoformat() - - # Handle pinning changes - if "pinned" in updates: - if updates["pinned"]: - await r.sadd("bulletin:pinned", post_id) - await r.zadd("bulletin:pinned_order", {post_id: post_dict.get("pin_order", 0)}) - else: - await r.srem("bulletin:pinned", post_id) - await r.zrem("bulletin:pinned_order", post_id) - - # Save updated post - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - - # Update Supabase - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("bulletin_posts").upsert(post_dict).execute() - except Exception as e: - logger.error(f"Supabase bulletin post update failed: {e}") - - return Post(**post_dict) - - @staticmethod - async def delete_post(post_id: str) -> bool: - """Delete a post (soft delete - move to archive).""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return False - - post_dict = json.loads(data) - post_dict["status"] = "archived" - post_dict["archived_at"] = datetime.utcnow().isoformat() - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - await r.srem("bulletin:status:published", post_id) - await r.srem("bulletin:status:draft", post_id) - await r.srem("bulletin:status:review", post_id) - await r.sadd("bulletin:status:archived", post_id) - await r.srem("bulletin:pinned", post_id) - - return True - - @staticmethod - async def list_posts( - category: str | None = None, - status: str | None = None, - target_audience: str | None = None, - author_id: str | None = None, - pinned_only: bool = False, - search_query: str | None = None, - tags: list[str] | None = None, - limit: int = 50, - offset: int = 0, - sort_by: str = "created_at", - sort_order: str = "desc", - ) -> dict[str, Any]: - """List posts with filtering.""" - r = await BulletinBoardManager._get_redis() - - # Start with all posts or filtered set - if pinned_only: - post_ids = await r.smembers("bulletin:pinned") - elif category: - post_ids = await r.smembers(f"bulletin:category:{category}") - elif status: - post_ids = await r.smembers(f"bulletin:status:{status}") - elif author_id: - post_ids = await r.smembers(f"bulletin:author:{author_id}") - elif target_audience: - post_ids = await r.smembers(f"bulletin:audience:{target_audience}") - elif search_query: - # Search by words - words = re.findall(r"\w+", search_query.lower()) - if words: - sets = [f"bulletin:search:{w}" for w in words if len(w) > 2] - if sets: - post_ids = await r.sinter(sets) - else: - post_ids = set() - else: - post_ids = set() - else: - all_posts = await r.hgetall("rmi:bulletin_posts") - post_ids = set(all_posts.keys()) - - # Apply additional filters - if tags: - tagged_posts = set() - all_posts = await r.hgetall("rmi:bulletin_posts") - for pid, data in all_posts.items(): - post_dict = json.loads(data) - if any(tag in post_dict.get("tags", []) for tag in tags): - tagged_posts.add(pid) - post_ids = post_ids.intersection(tagged_posts) - - # Fetch and sort posts - posts = [] - all_posts = await r.hgetall("rmi:bulletin_posts") - for pid in post_ids: - data = all_posts.get(pid) - if data: - post_dict = json.loads(data) - posts.append(post_dict) - - # Sort - reverse = sort_order == "desc" - posts.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse) - - # Pinned posts first if not pinned_only - if not pinned_only: - pinned_ids = await r.smembers("bulletin:pinned") - posts.sort( - key=lambda x: (x["post_id"] not in pinned_ids, x.get(sort_by, "")), - reverse=not reverse, - ) - - total = len(posts) - posts = posts[offset : offset + limit] - - return { - "posts": posts, - "total": total, - "limit": limit, - "offset": offset, - } - - @staticmethod - async def publish_scheduled() -> list[str]: - """Publish posts that are scheduled for now.""" - r = await BulletinBoardManager._get_redis() - now = int(time.time()) - - # Get scheduled posts that are due - due = await r.zrangebyscore("bulletin:scheduled", 0, now) - published = [] - - for post_id in due: - data = await r.hget("rmi:bulletin_posts", post_id) - if data: - post_dict = json.loads(data) - post_dict["status"] = "published" - post_dict["published_at"] = datetime.utcnow().isoformat() - post_dict["scheduled_at"] = None - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - await r.srem("bulletin:status:scheduled", post_id) - await r.sadd("bulletin:status:published", post_id) - await r.zrem("bulletin:scheduled", post_id) - - published.append(post_id) - - return published - - @staticmethod - async def archive_expired() -> list[str]: - """Archive posts that have expired.""" - r = await BulletinBoardManager._get_redis() - now = datetime.utcnow().isoformat() - - all_posts = await r.hgetall("rmi:bulletin_posts") - archived = [] - - for post_id, data in all_posts.items(): - post_dict = json.loads(data) - if post_dict.get("status") == "published" and post_dict.get("expires_at"): - if post_dict["expires_at"] < now: - post_dict["status"] = "archived" - post_dict["archived_at"] = now - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - await r.srem("bulletin:status:published", post_id) - await r.sadd("bulletin:status:archived", post_id) - await r.srem("bulletin:pinned", post_id) - - archived.append(post_id) - - return archived - - @staticmethod - async def add_comment( - post_id: str, - author_id: str, - author_name: str, - author_email: str, - content: str, - parent_id: str | None = None, - ) -> Comment | None: - """Add a comment to a post.""" - r = await BulletinBoardManager._get_redis() - - # Check if post exists and allows comments - post_data = await r.hget("rmi:bulletin_posts", post_id) - if not post_data: - return None - - post_dict = json.loads(post_data) - if not post_dict.get("allow_comments", False): - return None - - comment_id = f"comment_{int(time.time())}_{os.urandom(4).hex()}" - now = datetime.utcnow().isoformat() - - comment = Comment( - comment_id=comment_id, - post_id=post_id, - author_id=author_id, - author_name=author_name, - author_email=author_email, - content=ContentSanitizer.sanitize(content)[:2000], - created_at=now, - updated_at=now, - parent_id=parent_id, - ) - - await r.hset("rmi:bulletin_comments", comment_id, json.dumps(comment.to_dict())) - await r.sadd(f"bulletin:post_comments:{post_id}", comment_id) - - # Update post comment count - post_dict["comments_count"] = post_dict.get("comments_count", 0) + 1 - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - - return comment - - @staticmethod - async def get_comments(post_id: str, limit: int = 100) -> list[dict]: - """Get comments for a post.""" - r = await BulletinBoardManager._get_redis() - comment_ids = await r.smembers(f"bulletin:post_comments:{post_id}") - - comments = [] - for cid in comment_ids: - data = await r.hget("rmi:bulletin_comments", cid) - if data: - comments.append(json.loads(data)) - - comments.sort(key=lambda x: x.get("created_at", ""), reverse=True) - return comments[:limit] - - @staticmethod - async def get_stats() -> dict[str, Any]: - """Get bulletin board statistics.""" - r = await BulletinBoardManager._get_redis() - - stats = { - "total_posts": await r.hlen("rmi:bulletin_posts") or 0, - "published": await r.scard("bulletin:status:published") or 0, - "drafts": await r.scard("bulletin:status:draft") or 0, - "review": await r.scard("bulletin:status:review") or 0, - "archived": await r.scard("bulletin:status:archived") or 0, - "scheduled": await r.scard("bulletin:status:scheduled") or 0, - "pinned": await r.scard("bulletin:pinned") or 0, - "total_comments": await r.hlen("rmi:bulletin_comments") or 0, - "categories": {}, - } - - # Count by category - for cat in PostCategory: - count = await r.scard(f"bulletin:category:{cat.value}") or 0 - stats["categories"][cat.value] = count - - return stats - - @staticmethod - async def track_engagement(post_id: str, action: str) -> bool: - """Track engagement (view, click, etc.).""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return False - - post_dict = json.loads(data) - - if action == "view": - post_dict["view_count"] = post_dict.get("view_count", 0) + 1 - elif action == "click": - post_dict["click_count"] = post_dict.get("click_count", 0) + 1 - - # Calculate engagement score - views = post_dict.get("view_count", 0) - clicks = post_dict.get("click_count", 0) - post_dict["engagement_score"] = round((clicks / max(views, 1)) * 100, 2) - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - return True - - -# ═══════════════════════════════════════════════════════════ -# BADGES & X402 BOT PAYMENTS -# ═══════════════════════════════════════════════════════════ - -BADGES = { - "first_post": { - "name": "First Words", - "icon": "💬", - "desc": "Made your first post", - "tier": "bronze", - }, - "10_posts": {"name": "Chatterbox", "icon": "📢", "desc": "10 posts", "tier": "bronze"}, - "50_posts": {"name": "Board Regular", "icon": "🎙️", "desc": "50 posts", "tier": "silver"}, - "100_posts": { - "name": "Terminally Online", - "icon": "🖥️", - "desc": "100 posts - touch grass", - "tier": "gold", - }, - "10_upvotes": { - "name": "Approved", - "icon": "👍", - "desc": "10 upvotes on a post", - "tier": "bronze", - }, - "50_upvotes": {"name": "Crowd Favorite", "icon": "⭐", "desc": "50 upvotes", "tier": "silver"}, - "100_upvotes": {"name": "Legendary", "icon": "👑", "desc": "100 upvotes", "tier": "gold"}, - "rug_reporter": { - "name": "Rug Detective", - "icon": "🔍", - "desc": "Reported 3 verified rugs", - "tier": "silver", - }, - "scam_buster": { - "name": "Scam Buster", - "icon": "🛡️", - "desc": "10 scam alerts verified", - "tier": "gold", - }, - "honeypot_hunter": { - "name": "Honeypot Hunter", - "icon": "🍯", - "desc": "Found 5 honeypots", - "tier": "silver", - }, - "whale_watcher": { - "name": "Whale Watcher", - "icon": "🐋", - "desc": "Tracked 10 whale moves", - "tier": "silver", - }, - "alpha_caller": { - "name": "Alpha Caller", - "icon": "📈", - "desc": "Called 5 pumps", - "tier": "gold", - }, - "degen": { - "name": "Certified Degen", - "icon": "🎰", - "desc": "Posted in every category", - "tier": "gold", - }, - "rug_survivor": { - "name": "Rug Survivor", - "icon": "💀", - "desc": "Posted about getting rugged", - "tier": "bronze", - }, - "diamond_hands": { - "name": "Diamond Hands", - "icon": "💎", - "desc": "Held through -90%", - "tier": "diamond", - }, - "based": { - "name": "Based", - "icon": "🧠", - "desc": "Called a 10x before it happened", - "tier": "diamond", - }, - "bot_verified": { - "name": "Verified Bot", - "icon": "🤖", - "desc": "Registered x402 bot", - "tier": "silver", - }, - "bot_pro": {"name": "Bot Pro", "icon": "⚡", "desc": "100+ API calls", "tier": "gold"}, -} -BADGE_TIERS = {"bronze": "#CD7F32", "silver": "#C0C0C0", "gold": "#FFD700", "diamond": "#B9F2FF"} - - -async def get_user_badges(user_id: str) -> list: - try: - r = await BulletinBoardManager._get_redis() - ids = await r.smembers(f"bb:badges:{user_id}") - await r.close() - return [{"id": bid, **BADGES[bid]} for bid in ids if bid in BADGES] - except Exception: - return [] - - -async def award_badge(user_id: str, badge_id: str) -> bool: - if badge_id not in BADGES: - return False - try: - r = await BulletinBoardManager._get_redis() - ok = await r.sadd(f"bb:badges:{user_id}", badge_id) - await r.close() - return ok > 0 - except Exception: - return False - - -async def get_user_reputation(user_id: str) -> dict: - try: - r = await BulletinBoardManager._get_redis() - p = r.pipeline() - p.get(f"bb:rep:{user_id}") - p.scard(f"bb:badges:{user_id}") - p.get(f"bb:posts:{user_id}") - rep, bc, pc = await p.execute() - await r.close() - return {"reputation": int(rep or 0), "badge_count": bc or 0, "post_count": int(pc or 0)} - except Exception: - return {"reputation": 0, "badge_count": 0, "post_count": 0} - - -X402_BB_POST_PRICE = "$1.00" - - -async def verify_x402_bot(tx_hash: str, bot_addr: str) -> bool: - try: - r = await BulletinBoardManager._get_redis() - if await r.get(f"bb:x402:tx:{tx_hash}"): - await r.close() - return False - await r.setex(f"bb:x402:tx:{tx_hash}", 86400, bot_addr) - await r.setex(f"bb:x402:bot:{bot_addr}", 2592000, "1") # 30 day auth - await r.close() - return True - except Exception: - return False +from app.domains.bulletin import ( # noqa: F401 + BulletinBoardManager, + Comment, + ContentSanitizer, + Post, + PostCategory, + PostStatus, + Priority, + TargetAudience, + award_badge, + get_user_badges, + get_user_reputation, + verify_x402_bot, +) diff --git a/app/core/redis.py b/app/core/redis.py index d2e3e22..19a47cf 100644 --- a/app/core/redis.py +++ b/app/core/redis.py @@ -73,7 +73,7 @@ def get_redis_async() -> "aioredis.Redis | None": return None -def close_redis(): +async def close_redis() -> None: """Close Redis connection pool.""" global _sync_pool, _async_pool if _sync_pool: @@ -85,7 +85,7 @@ def close_redis(): _sync_pool = None if _async_pool: try: - _async_pool.disconnect() + await _async_pool.disconnect() logger.info("Async Redis connection pool closed") except Exception as e: logger.error(f"Async Redis pool close failed: {e}") diff --git a/app/domains/admin/__init__.py b/app/domains/admin/__init__.py new file mode 100644 index 0000000..02af84b --- /dev/null +++ b/app/domains/admin/__init__.py @@ -0,0 +1,31 @@ +"""Admin domain - public API. + +Phase 4 domain consolidation: app.admin_backend -> app.domains.admin. +""" +from __future__ import annotations + +from app.domains.admin.core import ( + PERMISSIONS, + AdminRole, + AdminUserStore, + AuditLogEntry, + AuditLogger, + SecurityManager, + SessionManager, + SystemHealthMonitor, + has_permission, + require_admin, +) + +__all__ = [ + "PERMISSIONS", + "AdminRole", + "AdminUserStore", + "AuditLogEntry", + "AuditLogger", + "SecurityManager", + "SessionManager", + "SystemHealthMonitor", + "has_permission", + "require_admin", +] diff --git a/app/domains/admin/core.py b/app/domains/admin/core.py new file mode 100644 index 0000000..dfb357e --- /dev/null +++ b/app/domains/admin/core.py @@ -0,0 +1,1070 @@ +""" +RMI Admin Backend - Complete Administration System +=================================================== +A hardened, feature-rich admin panel for the RugMunch Intelligence Platform. + +Features: + • Role-Based Access Control (RBAC) - superadmin, admin, moderator, viewer + • Audit Logging - every action logged with IP, timestamp, before/after state + • Rate Limiting - per-endpoint, per-user, per-IP limits + • IP Blocking / Allowlisting - ban bad actors by IP + • 2FA Support - TOTP-based two-factor authentication + • Session Management - active sessions, force logout, expiry control + • System Health - real-time metrics, disk, memory, CPU, service status + • User Management - CRUD, ban/unban, role assignment, activity tracking + • Configuration Management - env vars, feature flags, system settings + • Security Dashboard - failed logins, blocked IPs, threat alerts + • Content Management - announcements, blog posts, SEO settings + • Financial Dashboard - x402 revenue, payment tracking, analytics + • API Key Management - rotate, revoke, scope-limited keys + • Webhook Management - configure, test, monitor webhooks + • Backup & Restore - database, config, snapshot management + +Security: + - All endpoints require admin authentication (JWT + role check) + - Audit log of every admin action (immutable, append-only) + - Rate limiting on all admin endpoints (stricter than public) + - IP allowlist for admin access (optional) + - Session timeout and concurrent session limits + - Password policy enforcement for admin accounts + - Automatic lockout after failed login attempts +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import secrets +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from enum import StrEnum +from typing import Any + +from fastapi import HTTPException, Request + +logger = logging.getLogger("rmi_admin_backend") + + +# ── Role Definitions ────────────────────────────────────────── + + +class AdminRole(StrEnum): + """Admin role hierarchy. Higher = more permissions.""" + + SUPERADMIN = "superadmin" # Full access, can manage other admins + ADMIN = "admin" # Full access except admin management + MODERATOR = "moderator" # Content + user management, no system config + VIEWER = "viewer" # Read-only access to dashboards + SUPPORT = "support" # User management only, read-only system + + +# Permission matrix: role -> set of allowed permissions +PERMISSIONS = { + AdminRole.SUPERADMIN: { + "*", # All permissions + }, + AdminRole.ADMIN: { + "dashboard.read", + "users.read", + "users.write", + "users.ban", + "content.read", + "content.write", + "system.read", + "system.write", + "security.read", + "security.write", + "financial.read", + "financial.write", + "api_keys.read", + "api_keys.write", + "webhooks.read", + "webhooks.write", + "backups.read", + "backups.write", + "settings.read", + "settings.write", + "logs.read", + "analytics.read", + "token_deploy.read", + "token_deploy.write", + }, + AdminRole.MODERATOR: { + "dashboard.read", + "users.read", + "users.write", + "users.ban", + "content.read", + "content.write", + "security.read", + "logs.read", + "analytics.read", + }, + AdminRole.VIEWER: { + "dashboard.read", + "users.read", + "content.read", + "system.read", + "security.read", + "financial.read", + "analytics.read", + "logs.read", + }, + AdminRole.SUPPORT: { + "dashboard.read", + "users.read", + "users.write", + "content.read", + "logs.read", + "analytics.read", + }, +} + + +def has_permission(role: AdminRole, permission: str) -> bool: + """Check if a role has a specific permission.""" + if role == AdminRole.SUPERADMIN: + return True + perms = PERMISSIONS.get(role, set()) + return permission in perms or "*" in perms + + +# ── Audit Log ─────────────────────────────────────────────────── + + +@dataclass +class AuditLogEntry: + """Single audit log entry.""" + + entry_id: str + timestamp: str + admin_id: str + admin_email: str + action: str # e.g., "user.ban", "token.deploy", "config.update" + resource_type: str # e.g., "user", "token", "config" + resource_id: str + ip_address: str + user_agent: str + before_state: dict | None = None + after_state: dict | None = None + status: str = "success" # success, failed, denied + reason: str = "" + session_id: str = "" + request_id: str = "" + + def to_dict(self) -> dict: + return asdict(self) + + +class AuditLogger: + """ + Immutable audit logging system. + Stores to Redis (time-series) + file backup + optional Supabase. + """ + + @staticmethod + async def log( + admin_id: str, + admin_email: str, + action: str, + resource_type: str, + resource_id: str, + ip_address: str = "", + user_agent: str = "", + before_state: dict | None = None, + after_state: dict | None = None, + status: str = "success", + reason: str = "", + session_id: str = "", + request_id: str = "", + ) -> bool: + """Log an admin action.""" + entry = AuditLogEntry( + entry_id=f"audit_{int(time.time() * 1000)}_{secrets.token_hex(4)}", + timestamp=datetime.utcnow().isoformat(), + admin_id=admin_id, + admin_email=admin_email, + action=action, + resource_type=resource_type, + resource_id=resource_id, + ip_address=ip_address, + user_agent=user_agent, + before_state=before_state, + after_state=after_state, + status=status, + reason=reason, + session_id=session_id, + request_id=request_id, + ) + + # Write to Redis (time-series list) + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Add to time-series list + key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" + await r.lpush(key, json.dumps(entry.to_dict())) + + # Keep only last 30 days in Redis + await r.ltrim(key, 0, 9999) + + # Add to admin-specific log + await r.lpush(f"audit_log:admin:{admin_id}", json.dumps(entry.to_dict())) + + # Add to action-specific index + await r.sadd(f"audit_log:actions:{action}", entry.entry_id) + + except Exception as e: + logger.error(f"Redis audit log failed: {e}") + + # Write to local file (append-only, immutable) + try: + log_file = f"/var/log/rmi/audit_{datetime.utcnow().strftime('%Y-%m')}.jsonl" + os.makedirs(os.path.dirname(log_file), exist_ok=True) + with open(log_file, "a") as f: + f.write(json.dumps(entry.to_dict()) + "\n") + except Exception as e: + logger.error(f"File audit log failed: {e}") + + # Write to Supabase if available + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("audit_logs").insert(entry.to_dict()).execute() + except Exception as e: + logger.error(f"Supabase audit log failed: {e}") + + return True + + @staticmethod + async def query( + admin_id: str | None = None, + action: str | None = None, + resource_type: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + limit: int = 100, + offset: int = 0, + ) -> list[AuditLogEntry]: + """Query audit logs.""" + entries = [] + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Get today's log + key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" + logs = await r.lrange(key, offset, offset + limit - 1) + + for log in logs: + data = json.loads(log) + + # Filter + if admin_id and data.get("admin_id") != admin_id: + continue + if action and data.get("action") != action: + continue + if resource_type and data.get("resource_type") != resource_type: + continue + + entries.append(AuditLogEntry(**data)) + + except Exception as e: + logger.error(f"Audit query failed: {e}") + + return entries + + +# ── Security Manager ────────────────────────────────────────── + + +class SecurityManager: + """ + Security hardening for admin backend. + - Rate limiting + - IP blocking + - Failed login tracking + - Session management + """ + + # Rate limits: (max_requests, window_seconds) + RATE_LIMITS = { # noqa: RUF012 + "default": (60, 60), # 60/minute + "login": (5, 300), # 5/5min (strict for login) + "admin_write": (30, 60), # 30/minute for write ops + "token_deploy": (10, 3600), # 10/hour for token deploy + "snapshot": (20, 3600), # 20/hour for snapshots + "airdrop": (5, 3600), # 5/hour for airdrops + } + + @staticmethod + async def check_rate_limit( + identifier: str, + limit_type: str = "default", + ) -> dict[str, Any]: + """Check if request is within rate limit.""" + max_req, window = SecurityManager.RATE_LIMITS.get(limit_type, (60, 60)) + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + key = f"rate_limit:{limit_type}:{identifier}" + now = int(time.time()) + + # Clean old entries + await r.zremrangebyscore(key, 0, now - window) + + # Count current requests + count = await r.zcard(key) + + if count >= max_req: + # Get time until reset + oldest = await r.zrange(key, 0, 0, withscores=True) + reset_at = oldest[0][1] + window if oldest else now + window + + return { + "allowed": False, + "remaining": 0, + "reset_at": reset_at, + "limit": max_req, + "window": window, + } + + # Add current request + await r.zadd(key, {str(now): now}) + await r.expire(key, window) + + return { + "allowed": True, + "remaining": max_req - count - 1, + "reset_at": now + window, + "limit": max_req, + "window": window, + } + + except Exception as e: + logger.error(f"Rate limit check failed: {e}") + # Fail open (allow request) if Redis is down + return {"allowed": True, "remaining": -1, "reset_at": 0} + + @staticmethod + async def block_ip(ip: str, reason: str = "", duration_hours: int = 24) -> bool: + """Block an IP address.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + key = f"blocked_ip:{ip}" + data = { + "ip": ip, + "reason": reason, + "blocked_at": datetime.utcnow().isoformat(), + "expires_at": (datetime.utcnow() + timedelta(hours=duration_hours)).isoformat(), + "duration_hours": duration_hours, + } + + await r.setex(key, duration_hours * 3600, json.dumps(data)) + await r.sadd("blocked_ips:all", ip) + + logger.warning(f"IP blocked: {ip} for {duration_hours}h - {reason}") + return True + + except Exception as e: + logger.error(f"IP block failed: {e}") + return False + + @staticmethod + async def unblock_ip(ip: str) -> bool: + """Unblock an IP address.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + await r.delete(f"blocked_ip:{ip}") + await r.srem("blocked_ips:all", ip) + + return True + + except Exception as e: + logger.error(f"IP unblock failed: {e}") + return False + + @staticmethod + async def is_ip_blocked(ip: str) -> dict | None: + """Check if IP is blocked. Returns block info or None.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + data = await r.get(f"blocked_ip:{ip}") + if data: + return json.loads(data) + + return None + + except Exception as e: + logger.error(f"IP block check failed: {e}") + return None + + @staticmethod + async def track_failed_login(identifier: str) -> int: + """Track failed login attempts. Returns current count.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + key = f"failed_logins:{identifier}" + count = await r.incr(key) + await r.expire(key, 3600) # 1 hour window + + # Auto-block after 5 failed attempts + if count >= 5: + ip = identifier.split(":")[-1] if ":" in identifier else identifier + await SecurityManager.block_ip(ip, "Too many failed login attempts", 1) + + return count + + except Exception as e: + logger.error(f"Failed login tracking error: {e}") + return 0 + + @staticmethod + async def reset_failed_login(identifier: str) -> bool: + """Reset failed login counter (on successful login).""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + await r.delete(f"failed_logins:{identifier}") + return True + + except Exception as e: + logger.error(f"Reset failed login error: {e}") + return False + + +# ── Session Manager ─────────────────────────────────────────── + + +class SessionManager: + """ + Admin session management. + - Track active sessions + - Force logout + - Concurrent session limits + - Session expiry + """ + + MAX_CONCURRENT_SESSIONS = 3 + SESSION_TIMEOUT_HOURS = 8 + + @staticmethod + async def create_session( + admin_id: str, + admin_email: str, + role: AdminRole, + ip_address: str, + user_agent: str, + ) -> str: + """Create a new admin session.""" + session_id = f"sess_{secrets.token_urlsafe(16)}" + + session_data = { + "session_id": session_id, + "admin_id": admin_id, + "admin_email": admin_email, + "role": role.value if isinstance(role, AdminRole) else role, + "ip_address": ip_address, + "user_agent": user_agent, + "created_at": datetime.utcnow().isoformat(), + "expires_at": (datetime.utcnow() + timedelta(hours=SessionManager.SESSION_TIMEOUT_HOURS)).isoformat(), + "last_active": datetime.utcnow().isoformat(), + "is_active": True, + } + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Store session + key = f"admin_session:{session_id}" + await r.setex(key, SessionManager.SESSION_TIMEOUT_HOURS * 3600, json.dumps(session_data)) + + # Add to admin's session list + await r.sadd(f"admin_sessions:{admin_id}", session_id) + + # Enforce max concurrent sessions + sessions = await r.smembers(f"admin_sessions:{admin_id}") + if len(sessions) > SessionManager.MAX_CONCURRENT_SESSIONS: + # Remove oldest sessions + sorted_sessions = sorted(sessions) + for old_session in sorted_sessions[: -SessionManager.MAX_CONCURRENT_SESSIONS]: + await SessionManager.destroy_session(old_session) + + return session_id + + except Exception as e: + logger.error(f"Session creation failed: {e}") + return "" + + @staticmethod + async def validate_session(session_id: str) -> dict | None: + """Validate and refresh a session.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + data = await r.get(f"admin_session:{session_id}") + if not data: + return None + + session = json.loads(data) + + # Check expiry + expires = datetime.fromisoformat(session["expires_at"]) + if datetime.utcnow() > expires: + await SessionManager.destroy_session(session_id) + return None + + # Update last active + session["last_active"] = datetime.utcnow().isoformat() + await r.setex( + f"admin_session:{session_id}", + SessionManager.SESSION_TIMEOUT_HOURS * 3600, + json.dumps(session), + ) + + return session + + except Exception as e: + logger.error(f"Session validation failed: {e}") + return None + + @staticmethod + async def destroy_session(session_id: str) -> bool: + """Destroy a session (logout).""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Get session data for admin_id + data = await r.get(f"admin_session:{session_id}") + if data: + session = json.loads(data) + admin_id = session.get("admin_id") + if admin_id: + await r.srem(f"admin_sessions:{admin_id}", session_id) + + await r.delete(f"admin_session:{session_id}") + return True + + except Exception as e: + logger.error(f"Session destroy failed: {e}") + return False + + @staticmethod + async def get_active_sessions(admin_id: str) -> list[dict]: + """Get all active sessions for an admin.""" + sessions = [] + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + session_ids = await r.smembers(f"admin_sessions:{admin_id}") + for sid in session_ids: + data = await r.get(f"admin_session:{sid}") + if data: + session = json.loads(data) + # Check if still valid + expires = datetime.fromisoformat(session["expires_at"]) + if datetime.utcnow() < expires: + sessions.append(session) + else: + await r.srem(f"admin_sessions:{admin_id}", sid) + + except Exception as e: + logger.error(f"Get active sessions failed: {e}") + + return sessions + + @staticmethod + async def destroy_all_sessions(admin_id: str) -> int: + """Destroy all sessions for an admin (force logout everywhere).""" + count = 0 + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + session_ids = await r.smembers(f"admin_sessions:{admin_id}") + for sid in session_ids: + await r.delete(f"admin_session:{sid}") + count += 1 + + await r.delete(f"admin_sessions:{admin_id}") + + except Exception as e: + logger.error(f"Destroy all sessions failed: {e}") + + return count + + +# ── Admin User Store ────────────────────────────────────────── + + +class AdminUserStore: + """ + Store and manage admin user accounts. + Uses Redis as primary + Supabase as backup. + """ + + @staticmethod + async def get_admin(admin_id: str) -> dict | None: + """Get admin by ID.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + data = await r.hget("rmi:admins", admin_id) + if data: + return json.loads(data) + + return None + + except Exception as e: + logger.error(f"Get admin failed: {e}") + return None + + @staticmethod + async def get_admin_by_email(email: str) -> dict | None: + """Get admin by email.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + admin_id = await r.hget("rmi:admins:email", email.lower()) + if admin_id: + return await AdminUserStore.get_admin(admin_id) + + return None + + except Exception as e: + logger.error(f"Get admin by email failed: {e}") + return None + + @staticmethod + async def save_admin(admin: dict) -> bool: + """Save admin user.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + await r.hset("rmi:admins", admin["id"], json.dumps(admin)) + await r.hset("rmi:admins:email", admin["email"].lower(), admin["id"]) + + # Also save to Supabase + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("admin_users").upsert(admin).execute() + except Exception: + pass + + return True + + except Exception as e: + logger.error(f"Save admin failed: {e}") + return False + + @staticmethod + async def list_admins() -> list[dict]: + """List all admin users.""" + admins = [] + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + all_admins = await r.hgetall("rmi:admins") + for data in all_admins.values(): + admins.append(json.loads(data)) + + except Exception as e: + logger.error(f"List admins failed: {e}") + + return admins + + @staticmethod + async def create_admin( + email: str, + password: str, + role: AdminRole = AdminRole.VIEWER, + created_by: str = "", + ) -> dict | None: + """Create a new admin user.""" + from app.domains.auth.passwords import hash_password + + # Check if email already exists + existing = await AdminUserStore.get_admin_by_email(email) + if existing: + return None + + admin_id = hashlib.sha256(email.lower().encode()).hexdigest()[:16] + + admin = { + "id": admin_id, + "email": email, + "password_hash": hash_password(password), + "role": role.value, + "is_active": True, + "created_at": datetime.utcnow().isoformat(), + "created_by": created_by, + "last_login": None, + "login_count": 0, + "two_factor_enabled": False, + "two_factor_secret": None, + "ip_allowlist": [], + "metadata": {}, + } + + await AdminUserStore.save_admin(admin) + + # Remove password hash from returned dict + safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} + return safe_admin + + @staticmethod + async def verify_admin_login(email: str, password: str) -> dict | None: + """Verify admin login credentials.""" + from app.domains.auth.passwords import verify_password + + admin = await AdminUserStore.get_admin_by_email(email) + if not admin: + return None + + if not admin.get("is_active", True): + return None + + if not verify_password(password, admin.get("password_hash", "")): + return None + + # Update last login + admin["last_login"] = datetime.utcnow().isoformat() + admin["login_count"] = admin.get("login_count", 0) + 1 + await AdminUserStore.save_admin(admin) + + # Remove sensitive data + safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} + return safe_admin + + +# ── System Health Monitor ───────────────────────────────────── + + +class SystemHealthMonitor: + """ + Monitor system health and performance. + """ + + @staticmethod + async def get_system_health() -> dict[str, Any]: + """Get comprehensive system health.""" + import psutil + + # CPU + cpu_percent = psutil.cpu_percent(interval=0.5) + cpu_count = psutil.cpu_count() + + # Memory + memory = psutil.virtual_memory() + + # Disk + disk = psutil.disk_usage("/") + + # Network + net_io = psutil.net_io_counters() + + # Process info + backend_pid = os.getpid() + backend_proc = psutil.Process(backend_pid) + + # Check services + services = { + "redis": await SystemHealthMonitor._check_redis(), + "supabase": await SystemHealthMonitor._check_supabase(), + "backend": {"status": "running", "pid": backend_pid}, + } + + return { + "timestamp": datetime.utcnow().isoformat(), + "cpu": { + "percent": cpu_percent, + "count": cpu_count, + "per_cpu": psutil.cpu_percent(percpu=True, interval=0.1), + }, + "memory": { + "total_gb": round(memory.total / (1024**3), 2), + "available_gb": round(memory.available / (1024**3), 2), + "percent": memory.percent, + "used_gb": round(memory.used / (1024**3), 2), + }, + "disk": { + "total_gb": round(disk.total / (1024**3), 2), + "used_gb": round(disk.used / (1024**3), 2), + "free_gb": round(disk.free / (1024**3), 2), + "percent": round(disk.used / disk.total * 100, 1), + }, + "network": { + "bytes_sent_mb": round(net_io.bytes_sent / (1024**2), 2), + "bytes_recv_mb": round(net_io.bytes_recv / (1024**2), 2), + }, + "backend": { + "pid": backend_pid, + "memory_mb": round(backend_proc.memory_info().rss / (1024**2), 2), + "cpu_percent": backend_proc.cpu_percent(interval=0.2), + "threads": backend_proc.num_threads(), + "open_files": len(backend_proc.open_files()), + "connections": len(backend_proc.connections()), + }, + "services": services, + "uptime_seconds": int(time.time() - psutil.boot_time()), + } + + @staticmethod + async def _check_redis() -> dict: + """Check Redis connection.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + await r.ping() + info = await r.info() + return { + "status": "connected", + "version": info.get("redis_version", "unknown"), + "used_memory_mb": round(info.get("used_memory", 0) / (1024**2), 2), + "connected_clients": info.get("connected_clients", 0), + "total_keys": info.get("db0", {}).get("keys", 0) if isinstance(info.get("db0"), dict) else 0, + } + except Exception as e: + return {"status": "error", "error": str(e)} + + @staticmethod + async def _check_supabase() -> dict: + """Check Supabase connection.""" + try: + supabase_url = os.getenv("SUPABASE_URL") + if not supabase_url: + return {"status": "not_configured"} + + import httpx + + async with httpx.AsyncClient() as client: + r = await client.get(f"{supabase_url}/rest/v1/", timeout=5) + return { + "status": "connected" if r.status_code < 500 else "error", + "url": supabase_url, + "response_code": r.status_code, + } + except Exception as e: + return {"status": "error", "error": str(e)} + + +# ── Convenience: Admin Auth Decorator ───────────────────────── + + +async def require_admin( + request: Request, + required_permission: str = "", + min_role: AdminRole = AdminRole.VIEWER, +) -> dict[str, Any]: + """ + Verify admin authentication and authorization. + + Usage: + admin = await require_admin(request, "users.write", AdminRole.ADMIN) + """ + # Get session token from header + session_token = request.headers.get("X-Admin-Session", "") + if not session_token: + raise HTTPException(status_code=401, detail="Admin session required") + + # Validate session + session = await SessionManager.validate_session(session_token) + if not session: + raise HTTPException(status_code=401, detail="Invalid or expired session") + + # Check IP block + client_ip = request.client.host if request.client else "" + block_info = await SecurityManager.is_ip_blocked(client_ip) + if block_info: + raise HTTPException(status_code=403, detail="IP blocked") + + # Get admin + admin = await AdminUserStore.get_admin(session["admin_id"]) + if not admin or not admin.get("is_active", True): + raise HTTPException(status_code=403, detail="Admin account inactive") + + # Check role level + role = AdminRole(admin.get("role", "viewer")) + role_order = { + AdminRole.VIEWER: 0, + AdminRole.SUPPORT: 1, + AdminRole.MODERATOR: 2, + AdminRole.ADMIN: 3, + AdminRole.SUPERADMIN: 4, + } + if role_order[role] < role_order[min_role]: + raise HTTPException(status_code=403, detail="Insufficient privileges") + + # Check specific permission + if required_permission and not has_permission(role, required_permission): + raise HTTPException(status_code=403, detail="Permission denied") + + # Check rate limit + rate_check = await SecurityManager.check_rate_limit( + f"admin:{admin['id']}", + "admin_write" if required_permission.endswith(".write") else "default", + ) + if not rate_check["allowed"]: + raise HTTPException(status_code=429, detail=f"Rate limit exceeded. Reset at {rate_check['reset_at']}") + + # Log access + await AuditLogger.log( + admin_id=admin["id"], + admin_email=admin["email"], + action="admin.access", + resource_type="endpoint", + resource_id=str(request.url.path), + ip_address=client_ip, + user_agent=request.headers.get("user-agent", ""), + session_id=session_token, + ) + + return { + "admin": admin, + "session": session, + "role": role, + } diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index 4479e4d..deb69f7 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -5,6 +5,7 @@ Auth Router - Complete authentication system (email, wallet, OAuth, Telegram) import json import logging from datetime import datetime, timedelta +from typing import Any, cast from fastapi import APIRouter, HTTPException, Request @@ -77,7 +78,7 @@ router.include_router(oauth_router, tags=["auth"]) # ── Storage (Redis-backed user store) ── # ── Email Auth ── @router.post("/register", response_model=WalletAuthResponse) -def register_email(req: EmailRegisterRequest): +def register_email(req: EmailRegisterRequest) -> WalletAuthResponse: """Register a new user with email/password.""" if not _is_valid_email(req.email): raise HTTPException(status_code=400, detail="Invalid email format") @@ -93,7 +94,7 @@ def register_email(req: EmailRegisterRequest): user_id = _derive_user_id(req.email) hashed = hash_password(req.password) - user = { + user: dict[str, Any] = { "id": user_id, "email": req.email, "display_name": req.display_name, @@ -112,29 +113,33 @@ def register_email(req: EmailRegisterRequest): # Save user + email index r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", req.email.lower(), user_id) token = _create_jwt(user_id, req.email, "FREE", "USER", req.wallet_address) - return { - "access_token": token, - "refresh_token": token, - "user": { - "id": user_id, - "email": req.email, - "display_name": req.display_name, - "wallet_address": req.wallet_address, - "wallet_chain": req.wallet_chain, - "tier": "FREE", - "role": "USER", - "created_at": user["created_at"], + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user_id, + "email": req.email, + "display_name": req.display_name, + "wallet_address": req.wallet_address, + "wallet_chain": req.wallet_chain, + "tier": "FREE", + "role": "USER", + "created_at": user["created_at"], + }, }, - } + ) @router.post("/login", response_model=WalletAuthResponse) -def login_email(req: EmailLoginRequest): +def login_email(req: EmailLoginRequest) -> WalletAuthResponse: """Login with email/password. If 2FA enabled, returns partial token requiring /2fa/login.""" user = _get_user_by_email(req.email) if not user or not user.get("password_hash"): @@ -162,21 +167,24 @@ def login_email(req: EmailLoginRequest): payload["pre_auth"] = True pre_token = jose_jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) - return { - "access_token": pre_token, - "refresh_token": pre_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - "_2fa_required": True, + return cast( + "WalletAuthResponse", + { + "access_token": pre_token, + "refresh_token": pre_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + "_2fa_required": True, + }, }, - } + ) token = _create_jwt( user["id"], @@ -186,38 +194,37 @@ def login_email(req: EmailLoginRequest): user.get("wallet_address"), ) - return { - "access_token": token, - "refresh_token": token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) # ── Wallet Auth ── @router.post("/wallet/nonce", response_model=NonceResponse) -async def wallet_nonce(req: WalletNonceRequest): +async def wallet_nonce(req: WalletNonceRequest) -> NonceResponse: """Get a nonce for wallet signature.""" nonce = generate_nonce() timestamp = datetime.utcnow().isoformat() message = f"RugMunch Intelligence wants you to sign in with your {req.chain.title()} account.\n\nWallet: {req.address}\nNonce: {nonce}\nTimestamp: {timestamp}" - return { - "nonce": nonce, - "timestamp": timestamp, - "message": message, - } + return cast("NonceResponse", {"nonce": nonce, "timestamp": timestamp, "message": message}) @router.post("/wallet/verify", response_model=WalletAuthResponse) -async def wallet_verify(req: WalletVerifyRequest): +async def wallet_verify(req: WalletVerifyRequest) -> WalletAuthResponse: """Verify wallet signature and create session.""" from app.domains.auth.wallet import get_or_create_wallet_user, verify_wallet_signature @@ -226,25 +233,28 @@ async def wallet_verify(req: WalletVerifyRequest): user_data = await get_or_create_wallet_user(req.address) - return { - "access_token": user_data["access_token"], - "refresh_token": user_data["refresh_token"], - "user": { - "id": user_data["id"], - "email": user_data["email"], - "display_name": user_data.get("display_name", f"Agent {req.address[2:8].upper()}"), - "wallet_address": req.address, - "wallet_chain": req.chain, - "tier": user_data["tier"], - "role": user_data["role"], - "created_at": user_data["created_at"], + return cast( + "WalletAuthResponse", + { + "access_token": user_data["access_token"], + "refresh_token": user_data["refresh_token"], + "user": { + "id": user_data["id"], + "email": user_data["email"], + "display_name": user_data.get("display_name", f"Agent {req.address[2:8].upper()}"), + "wallet_address": req.address, + "wallet_chain": req.chain, + "tier": user_data["tier"], + "role": user_data["role"], + "created_at": user_data["created_at"], + }, }, - } + ) # ── User Profile ── @router.get("/user/me", response_model=UserResponse) -async def user_me(request: Request): +async def user_me(request: Request) -> UserResponse: """Get current authenticated user profile.""" auth_header = request.headers.get("Authorization", "") if not auth_header.startswith("Bearer "): @@ -259,26 +269,29 @@ async def user_me(request: Request): if not user: raise HTTPException(status_code=404, detail="User not found") - return { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - "xp": user.get("xp", 0), - "level": user.get("level", 1), - "badges": user.get("badges", []), - } + return cast( + "UserResponse", + { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + "xp": user.get("xp", 0), + "level": user.get("level", 1), + "badges": user.get("badges", []), + }, + ) # ── 2FA / TOTP Endpoints ── @router.post("/2fa/setup") -async def twofa_setup(request: Request): +async def twofa_setup(request: Request) -> TwoFASetupResponse: """Begin 2FA setup - generate secret + QR code. Returns plaintext secret + backup codes (shown once).""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -300,6 +313,7 @@ async def twofa_setup(request: Request): # Store pending secret (not enabled until verified) r = get_redis() + assert r is not None pending = { "secret": secret, "backup_codes": [_hash_backup_code(c) for c in backup_codes], @@ -307,16 +321,16 @@ async def twofa_setup(request: Request): } r.setex(f"rmi:2fa_pending:{user['id']}", timedelta(minutes=10), json.dumps(pending)) - return { + return cast("TwoFASetupResponse", { "secret": secret, "qr_code": qr_b64, "uri": uri, "backup_codes": backup_codes, # plaintext - shown once - } + }) @router.post("/2fa/enable") -async def twofa_enable(req: TwoFAEnableRequest, request: Request): +async def twofa_enable(req: TwoFAEnableRequest, request: Request) -> dict[str, Any]: """Enable 2FA - verify the first TOTP code, then persist.""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -326,6 +340,7 @@ async def twofa_enable(req: TwoFAEnableRequest, request: Request): raise HTTPException(status_code=401, detail="Authentication required") r = get_redis() + assert r is not None pending_raw = r.get(f"rmi:2fa_pending:{user['id']}") if not pending_raw: raise HTTPException(status_code=400, detail="No pending 2FA setup. Call /2fa/setup first.") @@ -357,7 +372,7 @@ async def twofa_enable(req: TwoFAEnableRequest, request: Request): @router.post("/2fa/disable") -async def twofa_disable(request: Request): +async def twofa_disable(request: Request) -> dict[str, Any]: """Disable 2FA - requires current password for security.""" user = await get_current_user(request) if not user: @@ -379,27 +394,27 @@ async def twofa_disable(request: Request): _save_user(user) logger.info(f"[2FA DISABLED] user={user['id']} email={user['email']}") - return {"status": "ok", "message": "2FA disabled successfully"} + return cast("dict[str, Any]", {"status": "ok", "message": "2FA disabled successfully"}) @router.get("/2fa/status", response_model=TwoFAStatusResponse) -async def twofa_status(request: Request): +async def twofa_status(request: Request) -> TwoFAStatusResponse: """Check 2FA status for current user.""" user = await get_current_user(request) if not user: raise HTTPException(status_code=401, detail="Authentication required") enabled = user.get("totp_enabled", False) - return { + return cast("TwoFAStatusResponse", { "enabled": enabled, "method": "totp" if enabled else "none", "created_at": user.get("totp_created_at"), "last_used": user.get("totp_last_used"), - } + }) @router.post("/2fa/verify") -async def twofa_verify(req: TwoFAVerifyRequest, request: Request): +async def twofa_verify(req: TwoFAVerifyRequest, request: Request) -> dict[str, Any]: """Verify a TOTP code (for re-auth during sensitive operations).""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -418,11 +433,11 @@ async def twofa_verify(req: TwoFAVerifyRequest, request: Request): user["totp_last_used"] = datetime.utcnow().isoformat() _save_user(user) - return {"status": "ok", "verified": True} + return cast("dict[str, Any]", {"status": "ok", "verified": True}) @router.post("/2fa/login") -async def twofa_login(req: TwoFALoginRequest): +async def twofa_login(req: TwoFALoginRequest) -> WalletAuthResponse: """Login with email + password + 2FA code. Returns full JWT on success.""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -473,17 +488,20 @@ async def twofa_login(req: TwoFALoginRequest): logger.info(f"[2FA LOGIN] user={user['id']} email={user['email']}") - return { - "access_token": token, - "refresh_token": token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) diff --git a/app/domains/auth/jwt.py b/app/domains/auth/jwt.py index 6d63428..b776b58 100644 --- a/app/domains/auth/jwt.py +++ b/app/domains/auth/jwt.py @@ -41,7 +41,7 @@ def _create_jwt( } if wallet: payload["wallet"] = wallet - return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) + return str(jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM)) def _verify_jwt(token: str) -> dict[str, Any] | None: diff --git a/app/domains/auth/oauth.py b/app/domains/auth/oauth.py index c2f4c27..812ed9b 100644 --- a/app/domains/auth/oauth.py +++ b/app/domains/auth/oauth.py @@ -7,8 +7,10 @@ from __future__ import annotations import json import os from datetime import datetime +from typing import cast from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import RedirectResponse from app.core.redis import get_redis from app.domains.auth.jwt import _create_jwt, _derive_user_id @@ -25,7 +27,7 @@ FRONTEND_URL = os.getenv("FRONTEND_URL", "https://rugmunch.io") @router.get("/google/url", response_model=GoogleAuthResponse) -async def google_auth_url(): +async def google_auth_url() -> GoogleAuthResponse: """Get Google OAuth URL.""" client_id = os.getenv("GOOGLE_CLIENT_ID") redirect_uri = os.getenv( @@ -33,7 +35,7 @@ async def google_auth_url(): ) if not client_id: - return {"url": "/auth/callback?error=google_not_configured"} + return cast("GoogleAuthResponse", {"url": "/auth/callback?error=google_not_configured"}) from urllib.parse import urlencode @@ -46,14 +48,12 @@ async def google_auth_url(): "prompt": "consent", } url = f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" - return {"url": url} + return cast("GoogleAuthResponse", {"url": url}) @router.get("/google/callback") -async def google_callback(request: Request): +async def google_callback(request: Request) -> RedirectResponse: """Handle Google OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - code = request.query_params.get("code") error = request.query_params.get("error") @@ -118,6 +118,7 @@ async def google_callback(request: Request): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) @@ -129,7 +130,7 @@ async def google_callback(request: Request): @router.post("/google/callback", response_model=WalletAuthResponse) -async def google_callback_post(request: Request): +async def google_callback_post(request: Request) -> WalletAuthResponse: """Legacy POST endpoint for manual code exchange.""" body = await request.json() code = body.get("code") @@ -192,6 +193,7 @@ async def google_callback_post(request: Request): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) @@ -199,24 +201,27 @@ async def google_callback_post(request: Request): user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") ) - return { - "access_token": jwt_token, - "refresh_token": jwt_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", email), - "wallet_address": None, - "wallet_chain": None, - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": jwt_token, + "refresh_token": jwt_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", email), + "wallet_address": None, + "wallet_chain": None, + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) @router.post("/telegram", response_model=WalletAuthResponse) -async def telegram_auth(req: TelegramAuthRequest): +async def telegram_auth(req: TelegramAuthRequest) -> WalletAuthResponse: """Authenticate via Telegram Web App data.""" bot_token = os.getenv("TELEGRAM_BOT_TOKEN") if not bot_token: @@ -272,6 +277,7 @@ async def telegram_auth(req: TelegramAuthRequest): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", telegram_user_id, json.dumps(user)) r.hset("rmi:users:telegram", str(req.id), telegram_user_id) @@ -279,24 +285,27 @@ async def telegram_auth(req: TelegramAuthRequest): user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") ) - return { - "access_token": jwt_token, - "refresh_token": jwt_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name"), - "wallet_address": None, - "wallet_chain": None, - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": jwt_token, + "refresh_token": jwt_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name"), + "wallet_address": None, + "wallet_chain": None, + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) @router.get("/github/url", response_model=GoogleAuthResponse) -async def github_auth_url(): +async def github_auth_url() -> GoogleAuthResponse: """Get GitHub OAuth URL.""" client_id = os.getenv("GITHUB_CLIENT_ID") redirect_uri = os.getenv( @@ -304,7 +313,7 @@ async def github_auth_url(): ) if not client_id: - return {"url": "/auth/callback?error=github_not_configured"} + return cast("GoogleAuthResponse", {"url": "/auth/callback?error=github_not_configured"}) from urllib.parse import urlencode @@ -314,14 +323,12 @@ async def github_auth_url(): "scope": "user:email", } url = f"https://github.com/login/oauth/authorize?{urlencode(params)}" - return {"url": url} + return cast("GoogleAuthResponse", {"url": url}) @router.get("/github/callback") -async def github_callback(request: Request): +async def github_callback(request: Request) -> RedirectResponse: """Handle GitHub OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - code = request.query_params.get("code") error = request.query_params.get("error") @@ -403,6 +410,7 @@ async def github_callback(request: Request): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) diff --git a/app/domains/auth/schemas.py b/app/domains/auth/schemas.py index 5a964f1..dce6f80 100644 --- a/app/domains/auth/schemas.py +++ b/app/domains/auth/schemas.py @@ -58,7 +58,7 @@ class UserResponse(BaseModel): created_at: str xp: int = 0 level: int = 1 - badges: list = [] + badges: list[str] = [] class GoogleAuthResponse(BaseModel): @@ -80,7 +80,7 @@ class TwoFASetupResponse(BaseModel): secret: str qr_code: str # base64 data URI uri: str # otpauth:// URI - backup_codes: list # plaintext - shown once + backup_codes: list[str] # plaintext - shown once class TwoFAEnableRequest(BaseModel): diff --git a/app/domains/auth/store.py b/app/domains/auth/store.py index 63be999..f398002 100644 --- a/app/domains/auth/store.py +++ b/app/domains/auth/store.py @@ -10,7 +10,7 @@ from __future__ import annotations import json import re -from typing import Any +from typing import Any, cast from app.core.redis import get_redis @@ -24,7 +24,7 @@ def _get_user(user_id: str) -> dict[str, Any] | None: if not raw: return None try: - return json.loads(raw) + return cast("dict[str, Any]", json.loads(raw)) except (json.JSONDecodeError, TypeError): return None diff --git a/app/domains/auth/totp.py b/app/domains/auth/totp.py index a84c95f..ff30494 100644 --- a/app/domains/auth/totp.py +++ b/app/domains/auth/totp.py @@ -41,7 +41,7 @@ def _generate_totp_secret() -> str: """Generate a new base32 TOTP secret.""" if not PYOTP_AVAILABLE: raise RuntimeError("pyotp not installed") - return pyotp.random_base32() + return str(pyotp.random_base32()) def _build_totp(secret: str) -> Any: @@ -54,13 +54,13 @@ def _verify_totp(secret: str, code: str) -> bool: if not PYOTP_AVAILABLE or not secret or not code: return False totp = _build_totp(secret) - return totp.verify(code, valid_window=TOTP_WINDOW) + return bool(totp.verify(code, valid_window=TOTP_WINDOW)) def _get_totp_uri(secret: str, email: str) -> str: """Get otpauth:// URI for QR code generation.""" totp = _build_totp(secret) - return totp.provisioning_uri(name=email, issuer_name=TOTP_ISSUER) + return str(totp.provisioning_uri(name=email, issuer_name=TOTP_ISSUER)) def _generate_qr_base64(uri: str) -> str: @@ -71,7 +71,7 @@ def _generate_qr_base64(uri: str) -> str: return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() -def _generate_backup_codes(count: int = 8) -> list: +def _generate_backup_codes(count: int = 8) -> list[str]: """Generate single-use backup codes.""" codes = [] for _ in range(count): diff --git a/app/domains/auth/wallet.py b/app/domains/auth/wallet.py index b4b222b..ebfeb76 100644 --- a/app/domains/auth/wallet.py +++ b/app/domains/auth/wallet.py @@ -33,7 +33,7 @@ def verify_wallet_signature(message: str, signature: str, address: str) -> bool: return len(signature) >= 60 -def decode_signature(signature: str) -> tuple: +def decode_signature(signature: str) -> tuple[str, str, int]: """ Decode a wallet signature into r, s, v components. diff --git a/app/domains/billing/x402/tools/integration.py b/app/domains/billing/x402/tools/integration.py index 59b1689..05bb131 100644 --- a/app/domains/billing/x402/tools/integration.py +++ b/app/domains/billing/x402/tools/integration.py @@ -388,7 +388,7 @@ async def framework_discovery(): "free": True, }, "mcp": { - "endpoint": "python -m app.mcp.x402_mcp_server", + "endpoint": "python -m app.domains.mcp.x402_mcp_server", "format": "Model Context Protocol (stdio)", "usage": "Claude Desktop, Claude Code, any MCP client", "free": True, diff --git a/app/domains/bulletin/__init__.py b/app/domains/bulletin/__init__.py new file mode 100644 index 0000000..d4a4b72 --- /dev/null +++ b/app/domains/bulletin/__init__.py @@ -0,0 +1,35 @@ +"""Bulletin domain - public API. + +Phase 4 domain consolidation: app.bulletin_board -> app.domains.bulletin. +""" +from __future__ import annotations + +from app.domains.bulletin.core import ( + BulletinBoardManager, + Comment, + ContentSanitizer, + Post, + PostCategory, + PostStatus, + Priority, + TargetAudience, + award_badge, + get_user_badges, + get_user_reputation, + verify_x402_bot, +) + +__all__ = [ + "BulletinBoardManager", + "Comment", + "ContentSanitizer", + "Post", + "PostCategory", + "PostStatus", + "Priority", + "TargetAudience", + "award_badge", + "get_user_badges", + "get_user_reputation", + "verify_x402_bot", +] diff --git a/app/domains/bulletin/core.py b/app/domains/bulletin/core.py new file mode 100644 index 0000000..3948113 --- /dev/null +++ b/app/domains/bulletin/core.py @@ -0,0 +1,902 @@ +""" +RMI Bulletin Board Management System +===================================== +A full content management backend for announcements, news, alerts, platform +communications, and community bulletin boards. + +Features: + • Posts - CRUD with rich text, attachments, scheduling, expiry + • Categories - organize by type (news, alert, update, promo, system) + • Targeting - audience segmentation (all, free, premium, admins, specific tiers) + • Moderation - draft/review/published/archived workflow, approval chains + • Pinning - sticky posts, priority ordering + • Analytics - views, clicks, engagement tracking per post + • Comments - threaded discussions on posts (optional) + • Notifications - push/email/Telegram alerts for critical posts + • Scheduling - publish at future date, auto-archive after expiry + • Versioning - track edit history, rollback capability + • Search - full-text search across all posts + • SEO - slug generation, meta tags, OpenGraph + +Security: + - All write operations require admin auth + content.write permission + - Audit log of every content change + - Rate limiting on publish operations + - Content sanitization (strip XSS, validate HTML) +""" + +from __future__ import annotations + +import html +import json +import logging +import os +import re +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime +from enum import StrEnum +from typing import Any, ClassVar + +logger = logging.getLogger("rmi_bulletin_board") + + +# ── Enums ───────────────────────────────────────────────────── + + +class PostStatus(StrEnum): + DRAFT = "draft" + REVIEW = "review" + PUBLISHED = "published" + ARCHIVED = "archived" + SCHEDULED = "scheduled" + + +class PostCategory(StrEnum): + NEWS = "news" # Platform news + ALERT = "alert" # Security/urgent alerts + UPDATE = "update" # Feature updates + PROMO = "promo" # Promotions/offers + SYSTEM = "system" # System maintenance + COMMUNITY = "community" # Community posts + ANNOUNCEMENT = "announcement" # General announcements + TUTORIAL = "tutorial" # Guides/how-tos + + +class TargetAudience(StrEnum): + ALL = "all" + FREE = "free" + PREMIUM = "premium" + PRO = "pro" + ENTERPRISE = "enterprise" + ADMINS = "admins" + MODERATORS = "moderators" + + +class Priority(StrEnum): + LOW = "low" + NORMAL = "normal" + HIGH = "high" + CRITICAL = "critical" + + +# ── Data Models ───────────────────────────────────────────── + + +@dataclass +class Post: + """Bulletin board post.""" + + post_id: str + title: str + slug: str + content: str + summary: str + category: str + status: str + priority: str + target_audience: str + author_id: str + author_email: str + author_name: str + created_at: str + updated_at: str + published_at: str | None = None + scheduled_at: str | None = None + expires_at: str | None = None + archived_at: str | None = None + pinned: bool = False + pin_order: int = 0 + featured_image: str = "" + attachments: list[dict] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + meta_title: str = "" + meta_description: str = "" + og_image: str = "" + view_count: int = 0 + click_count: int = 0 + engagement_score: float = 0.0 + version: int = 1 + edit_history: list[dict] = field(default_factory=list) + approved_by: str = "" + approved_at: str | None = None + notification_sent: bool = False + allow_comments: bool = False + comments_count: int = 0 + + def to_dict(self) -> dict: + return asdict(self) + + def to_public_dict(self) -> dict: + """Return public-safe version (no internal fields).""" + return { + "post_id": self.post_id, + "title": self.title, + "slug": self.slug, + "content": self.content, + "summary": self.summary, + "category": self.category, + "priority": self.priority, + "author_name": self.author_name, + "created_at": self.created_at, + "published_at": self.published_at, + "expires_at": self.expires_at, + "pinned": self.pinned, + "pin_order": self.pin_order, + "featured_image": self.featured_image, + "attachments": self.attachments, + "tags": self.tags, + "meta_title": self.meta_title, + "meta_description": self.meta_description, + "og_image": self.og_image, + "view_count": self.view_count, + "allow_comments": self.allow_comments, + "comments_count": self.comments_count, + } + + +@dataclass +class Comment: + """Comment on a post.""" + + comment_id: str + post_id: str + author_id: str + author_name: str + author_email: str + content: str + created_at: str + updated_at: str + parent_id: str | None = None + status: str = "approved" # approved, pending, rejected + likes: int = 0 + replies: list[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return asdict(self) + + +# ── Content Sanitizer ─────────────────────────────────────── + + +class ContentSanitizer: + """Sanitize user-generated content to prevent XSS.""" + + ALLOWED_TAGS: ClassVar[dict] ={ + "p", + "br", + "strong", + "b", + "em", + "i", + "u", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "ul", + "ol", + "li", + "a", + "img", + "blockquote", + "code", + "pre", + "table", + "thead", + "tbody", + "tr", + "td", + "th", + "div", + "span", + "hr", + "sub", + "sup", + "del", + "ins", + } + + ALLOWED_ATTRS: ClassVar[dict] ={ + "a": ["href", "title", "target"], + "img": ["src", "alt", "title", "width", "height"], + "div": ["class"], + "span": ["class"], + "code": ["class"], + "pre": ["class"], + } + + @staticmethod + def sanitize(text: str) -> str: + """Basic HTML sanitization.""" + if not text: + return "" + + # Escape HTML entities first + text = html.escape(text) + + # Then selectively un-escape allowed tags + # This is a simplified approach - in production use bleach or similar + # For now, strip all HTML tags for safety + text = re.sub(r"<[^>]+>", "", text) + + return text.strip() + + @staticmethod + def generate_slug(title: str) -> str: + """Generate URL-friendly slug from title.""" + slug = re.sub(r"[^\w\s-]", "", title.lower()) + slug = re.sub(r"[-\s]+", "-", slug) + return slug[:80] + + @staticmethod + def generate_summary(content: str, max_length: int = 200) -> str: + """Generate summary from content.""" + # Strip HTML + text = re.sub(r"<[^>]+>", "", content) + text = text.replace("\n", " ").strip() + if len(text) > max_length: + text = text[:max_length].rsplit(" ", 1)[0] + "..." + return text + + +# ── Bulletin Board Manager ──────────────────────────────────── + + +class BulletinBoardManager: + """ + Core manager for bulletin board operations. + Uses Redis as primary store + Supabase backup. + """ + + @staticmethod + async def _get_redis(): + import redis.asyncio as redis_lib + + return redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + @staticmethod + async def create_post( + title: str, + content: str, + category: str, + author_id: str, + author_email: str, + author_name: str = "", + priority: str = "normal", + target_audience: str = "all", + status: str = "draft", + featured_image: str = "", + attachments: list[dict] | None = None, + tags: list[str] | None = None, + scheduled_at: str | None = None, + expires_at: str | None = None, + pinned: bool = False, + allow_comments: bool = False, + meta_title: str = "", + meta_description: str = "", + ) -> Post: + """Create a new post.""" + post_id = f"post_{int(time.time())}_{os.urandom(4).hex()}" + slug = ContentSanitizer.generate_slug(title) + summary = ContentSanitizer.generate_summary(content) + now = datetime.utcnow().isoformat() + + # Sanitize content + content = ContentSanitizer.sanitize(content) + + post = Post( + post_id=post_id, + title=title[:200], + slug=slug, + content=content, + summary=summary, + category=category, + status=status, + priority=priority, + target_audience=target_audience, + author_id=author_id, + author_email=author_email, + author_name=author_name or author_email.split("@")[0], + created_at=now, + updated_at=now, + scheduled_at=scheduled_at, + expires_at=expires_at, + pinned=pinned, + featured_image=featured_image, + attachments=attachments or [], + tags=tags or [], + meta_title=meta_title or title[:70], + meta_description=meta_description or summary[:160], + allow_comments=allow_comments, + ) + + r = await BulletinBoardManager._get_redis() + + # Save post + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post.to_dict())) + + # Add to category index + await r.sadd(f"bulletin:category:{category}", post_id) + + # Add to status index + await r.sadd(f"bulletin:status:{status}", post_id) + + # Add to author index + await r.sadd(f"bulletin:author:{author_id}", post_id) + + # Add to audience index + await r.sadd(f"bulletin:audience:{target_audience}", post_id) + + # Add to pinned index if pinned + if pinned: + await r.sadd("bulletin:pinned", post_id) + await r.zadd("bulletin:pinned_order", {post_id: post.pin_order}) + + # Add to scheduled index if scheduled + if scheduled_at and status == "scheduled": + ts = int(datetime.fromisoformat(scheduled_at).timestamp()) + await r.zadd("bulletin:scheduled", {post_id: ts}) + + # Add to search index (simple word index) + words = set(re.findall(r"\w+", title.lower() + " " + content.lower())) + for word in words: + if len(word) > 2: + await r.sadd(f"bulletin:search:{word}", post_id) + + # Save to Supabase + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("bulletin_posts").insert(post.to_dict()).execute() + except Exception as e: + logger.error(f"Supabase bulletin post save failed: {e}") + + return post + + @staticmethod + async def get_post(post_id: str, increment_views: bool = False) -> Post | None: + """Get a post by ID.""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return None + + post_dict = json.loads(data) + + if increment_views and post_dict.get("status") == "published": + post_dict["view_count"] = post_dict.get("view_count", 0) + 1 + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + + return Post(**post_dict) + + @staticmethod + async def get_post_by_slug(slug: str) -> Post | None: + """Get a post by slug.""" + r = await BulletinBoardManager._get_redis() + # Search all posts for matching slug + all_posts = await r.hgetall("rmi:bulletin_posts") + for _post_id, data in all_posts.items(): + post_dict = json.loads(data) + if post_dict.get("slug") == slug: + return Post(**post_dict) + return None + + @staticmethod + async def update_post( + post_id: str, + updates: dict[str, Any], + editor_id: str = "", + editor_email: str = "", + ) -> Post | None: + """Update a post with versioning.""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return None + + post_dict = json.loads(data) + before_state = {k: post_dict.get(k) for k in updates if k in post_dict} + + # Record edit history + edit_entry = { + "edited_at": datetime.utcnow().isoformat(), + "edited_by": editor_id, + "editor_email": editor_email, + "changes": before_state, + "version": post_dict.get("version", 1), + } + + history = post_dict.get("edit_history", []) + history.append(edit_entry) + post_dict["edit_history"] = history + post_dict["version"] = post_dict.get("version", 1) + 1 + post_dict["updated_at"] = datetime.utcnow().isoformat() + + # Apply updates + for key, value in updates.items(): + if key == "content": + value = ContentSanitizer.sanitize(value) + post_dict["summary"] = ContentSanitizer.generate_summary(value) + if key == "title": + post_dict["slug"] = ContentSanitizer.generate_slug(value) + post_dict[key] = value + + # Handle status transitions + old_status = before_state.get("status") + new_status = post_dict.get("status") + if old_status != new_status: + # Update status indexes + if old_status: + await r.srem(f"bulletin:status:{old_status}", post_id) + await r.sadd(f"bulletin:status:{new_status}", post_id) + + if new_status == "published": + post_dict["published_at"] = datetime.utcnow().isoformat() + elif new_status == "archived": + post_dict["archived_at"] = datetime.utcnow().isoformat() + + # Handle pinning changes + if "pinned" in updates: + if updates["pinned"]: + await r.sadd("bulletin:pinned", post_id) + await r.zadd("bulletin:pinned_order", {post_id: post_dict.get("pin_order", 0)}) + else: + await r.srem("bulletin:pinned", post_id) + await r.zrem("bulletin:pinned_order", post_id) + + # Save updated post + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + + # Update Supabase + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("bulletin_posts").upsert(post_dict).execute() + except Exception as e: + logger.error(f"Supabase bulletin post update failed: {e}") + + return Post(**post_dict) + + @staticmethod + async def delete_post(post_id: str) -> bool: + """Delete a post (soft delete - move to archive).""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return False + + post_dict = json.loads(data) + post_dict["status"] = "archived" + post_dict["archived_at"] = datetime.utcnow().isoformat() + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + await r.srem("bulletin:status:published", post_id) + await r.srem("bulletin:status:draft", post_id) + await r.srem("bulletin:status:review", post_id) + await r.sadd("bulletin:status:archived", post_id) + await r.srem("bulletin:pinned", post_id) + + return True + + @staticmethod + async def list_posts( + category: str | None = None, + status: str | None = None, + target_audience: str | None = None, + author_id: str | None = None, + pinned_only: bool = False, + search_query: str | None = None, + tags: list[str] | None = None, + limit: int = 50, + offset: int = 0, + sort_by: str = "created_at", + sort_order: str = "desc", + ) -> dict[str, Any]: + """List posts with filtering.""" + r = await BulletinBoardManager._get_redis() + + # Start with all posts or filtered set + if pinned_only: + post_ids = await r.smembers("bulletin:pinned") + elif category: + post_ids = await r.smembers(f"bulletin:category:{category}") + elif status: + post_ids = await r.smembers(f"bulletin:status:{status}") + elif author_id: + post_ids = await r.smembers(f"bulletin:author:{author_id}") + elif target_audience: + post_ids = await r.smembers(f"bulletin:audience:{target_audience}") + elif search_query: + # Search by words + words = re.findall(r"\w+", search_query.lower()) + if words: + sets = [f"bulletin:search:{w}" for w in words if len(w) > 2] + if sets: + post_ids = await r.sinter(sets) + else: + post_ids = set() + else: + post_ids = set() + else: + all_posts = await r.hgetall("rmi:bulletin_posts") + post_ids = set(all_posts.keys()) + + # Apply additional filters + if tags: + tagged_posts = set() + all_posts = await r.hgetall("rmi:bulletin_posts") + for pid, data in all_posts.items(): + post_dict = json.loads(data) + if any(tag in post_dict.get("tags", []) for tag in tags): + tagged_posts.add(pid) + post_ids = post_ids.intersection(tagged_posts) + + # Fetch and sort posts + posts = [] + all_posts = await r.hgetall("rmi:bulletin_posts") + for pid in post_ids: + data = all_posts.get(pid) + if data: + post_dict = json.loads(data) + posts.append(post_dict) + + # Sort + reverse = sort_order == "desc" + posts.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse) + + # Pinned posts first if not pinned_only + if not pinned_only: + pinned_ids = await r.smembers("bulletin:pinned") + posts.sort( + key=lambda x: (x["post_id"] not in pinned_ids, x.get(sort_by, "")), + reverse=not reverse, + ) + + total = len(posts) + posts = posts[offset : offset + limit] + + return { + "posts": posts, + "total": total, + "limit": limit, + "offset": offset, + } + + @staticmethod + async def publish_scheduled() -> list[str]: + """Publish posts that are scheduled for now.""" + r = await BulletinBoardManager._get_redis() + now = int(time.time()) + + # Get scheduled posts that are due + due = await r.zrangebyscore("bulletin:scheduled", 0, now) + published = [] + + for post_id in due: + data = await r.hget("rmi:bulletin_posts", post_id) + if data: + post_dict = json.loads(data) + post_dict["status"] = "published" + post_dict["published_at"] = datetime.utcnow().isoformat() + post_dict["scheduled_at"] = None + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + await r.srem("bulletin:status:scheduled", post_id) + await r.sadd("bulletin:status:published", post_id) + await r.zrem("bulletin:scheduled", post_id) + + published.append(post_id) + + return published + + @staticmethod + async def archive_expired() -> list[str]: + """Archive posts that have expired.""" + r = await BulletinBoardManager._get_redis() + now = datetime.utcnow().isoformat() + + all_posts = await r.hgetall("rmi:bulletin_posts") + archived = [] + + for post_id, data in all_posts.items(): + post_dict = json.loads(data) + if post_dict.get("status") == "published" and post_dict.get("expires_at") and post_dict["expires_at"] < now: + post_dict["status"] = "archived" + post_dict["archived_at"] = now + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + await r.srem("bulletin:status:published", post_id) + await r.sadd("bulletin:status:archived", post_id) + await r.srem("bulletin:pinned", post_id) + + archived.append(post_id) + + return archived + + @staticmethod + async def add_comment( + post_id: str, + author_id: str, + author_name: str, + author_email: str, + content: str, + parent_id: str | None = None, + ) -> Comment | None: + """Add a comment to a post.""" + r = await BulletinBoardManager._get_redis() + + # Check if post exists and allows comments + post_data = await r.hget("rmi:bulletin_posts", post_id) + if not post_data: + return None + + post_dict = json.loads(post_data) + if not post_dict.get("allow_comments", False): + return None + + comment_id = f"comment_{int(time.time())}_{os.urandom(4).hex()}" + now = datetime.utcnow().isoformat() + + comment = Comment( + comment_id=comment_id, + post_id=post_id, + author_id=author_id, + author_name=author_name, + author_email=author_email, + content=ContentSanitizer.sanitize(content)[:2000], + created_at=now, + updated_at=now, + parent_id=parent_id, + ) + + await r.hset("rmi:bulletin_comments", comment_id, json.dumps(comment.to_dict())) + await r.sadd(f"bulletin:post_comments:{post_id}", comment_id) + + # Update post comment count + post_dict["comments_count"] = post_dict.get("comments_count", 0) + 1 + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + + return comment + + @staticmethod + async def get_comments(post_id: str, limit: int = 100) -> list[dict]: + """Get comments for a post.""" + r = await BulletinBoardManager._get_redis() + comment_ids = await r.smembers(f"bulletin:post_comments:{post_id}") + + comments = [] + for cid in comment_ids: + data = await r.hget("rmi:bulletin_comments", cid) + if data: + comments.append(json.loads(data)) + + comments.sort(key=lambda x: x.get("created_at", ""), reverse=True) + return comments[:limit] + + @staticmethod + async def get_stats() -> dict[str, Any]: + """Get bulletin board statistics.""" + r = await BulletinBoardManager._get_redis() + + stats = { + "total_posts": await r.hlen("rmi:bulletin_posts") or 0, + "published": await r.scard("bulletin:status:published") or 0, + "drafts": await r.scard("bulletin:status:draft") or 0, + "review": await r.scard("bulletin:status:review") or 0, + "archived": await r.scard("bulletin:status:archived") or 0, + "scheduled": await r.scard("bulletin:status:scheduled") or 0, + "pinned": await r.scard("bulletin:pinned") or 0, + "total_comments": await r.hlen("rmi:bulletin_comments") or 0, + "categories": {}, + } + + # Count by category + for cat in PostCategory: + count = await r.scard(f"bulletin:category:{cat.value}") or 0 + stats["categories"][cat.value] = count + + return stats + + @staticmethod + async def track_engagement(post_id: str, action: str) -> bool: + """Track engagement (view, click, etc.).""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return False + + post_dict = json.loads(data) + + if action == "view": + post_dict["view_count"] = post_dict.get("view_count", 0) + 1 + elif action == "click": + post_dict["click_count"] = post_dict.get("click_count", 0) + 1 + + # Calculate engagement score + views = post_dict.get("view_count", 0) + clicks = post_dict.get("click_count", 0) + post_dict["engagement_score"] = round((clicks / max(views, 1)) * 100, 2) + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + return True + + +# ═══════════════════════════════════════════════════════════ +# BADGES & X402 BOT PAYMENTS +# ═══════════════════════════════════════════════════════════ + +BADGES = { + "first_post": { + "name": "First Words", + "icon": "💬", + "desc": "Made your first post", + "tier": "bronze", + }, + "10_posts": {"name": "Chatterbox", "icon": "📢", "desc": "10 posts", "tier": "bronze"}, + "50_posts": {"name": "Board Regular", "icon": "🎙️", "desc": "50 posts", "tier": "silver"}, + "100_posts": { + "name": "Terminally Online", + "icon": "🖥️", + "desc": "100 posts - touch grass", + "tier": "gold", + }, + "10_upvotes": { + "name": "Approved", + "icon": "👍", + "desc": "10 upvotes on a post", + "tier": "bronze", + }, + "50_upvotes": {"name": "Crowd Favorite", "icon": "⭐", "desc": "50 upvotes", "tier": "silver"}, + "100_upvotes": {"name": "Legendary", "icon": "👑", "desc": "100 upvotes", "tier": "gold"}, + "rug_reporter": { + "name": "Rug Detective", + "icon": "🔍", + "desc": "Reported 3 verified rugs", + "tier": "silver", + }, + "scam_buster": { + "name": "Scam Buster", + "icon": "🛡️", + "desc": "10 scam alerts verified", + "tier": "gold", + }, + "honeypot_hunter": { + "name": "Honeypot Hunter", + "icon": "🍯", + "desc": "Found 5 honeypots", + "tier": "silver", + }, + "whale_watcher": { + "name": "Whale Watcher", + "icon": "🐋", + "desc": "Tracked 10 whale moves", + "tier": "silver", + }, + "alpha_caller": { + "name": "Alpha Caller", + "icon": "📈", + "desc": "Called 5 pumps", + "tier": "gold", + }, + "degen": { + "name": "Certified Degen", + "icon": "🎰", + "desc": "Posted in every category", + "tier": "gold", + }, + "rug_survivor": { + "name": "Rug Survivor", + "icon": "💀", + "desc": "Posted about getting rugged", + "tier": "bronze", + }, + "diamond_hands": { + "name": "Diamond Hands", + "icon": "💎", + "desc": "Held through -90%", + "tier": "diamond", + }, + "based": { + "name": "Based", + "icon": "🧠", + "desc": "Called a 10x before it happened", + "tier": "diamond", + }, + "bot_verified": { + "name": "Verified Bot", + "icon": "🤖", + "desc": "Registered x402 bot", + "tier": "silver", + }, + "bot_pro": {"name": "Bot Pro", "icon": "⚡", "desc": "100+ API calls", "tier": "gold"}, +} +BADGE_TIERS = {"bronze": "#CD7F32", "silver": "#C0C0C0", "gold": "#FFD700", "diamond": "#B9F2FF"} + + +async def get_user_badges(user_id: str) -> list: + try: + r = await BulletinBoardManager._get_redis() + ids = await r.smembers(f"bb:badges:{user_id}") + await r.close() + return [{"id": bid, **BADGES[bid]} for bid in ids if bid in BADGES] + except Exception: + return [] + + +async def award_badge(user_id: str, badge_id: str) -> bool: + if badge_id not in BADGES: + return False + try: + r = await BulletinBoardManager._get_redis() + ok = await r.sadd(f"bb:badges:{user_id}", badge_id) + await r.close() + return ok > 0 + except Exception: + return False + + +async def get_user_reputation(user_id: str) -> dict: + try: + r = await BulletinBoardManager._get_redis() + p = r.pipeline() + p.get(f"bb:rep:{user_id}") + p.scard(f"bb:badges:{user_id}") + p.get(f"bb:posts:{user_id}") + rep, bc, pc = await p.execute() + await r.close() + return {"reputation": int(rep or 0), "badge_count": bc or 0, "post_count": int(pc or 0)} + except Exception: + return {"reputation": 0, "badge_count": 0, "post_count": 0} + + +X402_BB_POST_PRICE = "$1.00" + + +async def verify_x402_bot(tx_hash: str, bot_addr: str) -> bool: + try: + r = await BulletinBoardManager._get_redis() + if await r.get(f"bb:x402:tx:{tx_hash}"): + await r.close() + return False + await r.setex(f"bb:x402:tx:{tx_hash}", 86400, bot_addr) + await r.setex(f"bb:x402:bot:{bot_addr}", 2592000, "1") # 30 day auth + await r.close() + return True + except Exception: + return False diff --git a/app/domains/databus/provider_implementations.py b/app/domains/databus/provider_implementations.py index 48f5a12..88e3a91 100644 --- a/app/domains/databus/provider_implementations.py +++ b/app/domains/databus/provider_implementations.py @@ -183,7 +183,10 @@ async def _passthrough_news(**kwargs) -> dict | None: async def _passthrough_alerts(**kwargs) -> dict | None: """Alerts from our real alert pipeline.""" try: - from app.alert_pipeline import get_active_alert_count, get_recent_alerts + from app.domains.intelligence.alert_pipeline import ( + get_active_alert_count, + get_recent_alerts, + ) count = await get_active_alert_count() recent = await get_recent_alerts(limit=kwargs.get("limit", 20)) diff --git a/app/domains/databus/providers/__init__.py b/app/domains/databus/providers/__init__.py index 3525891..e7d644e 100644 --- a/app/domains/databus/providers/__init__.py +++ b/app/domains/databus/providers/__init__.py @@ -362,7 +362,10 @@ async def _passthrough_news(**kwargs) -> dict | None: async def _passthrough_alerts(**kwargs) -> dict | None: """Alerts from our real alert pipeline.""" try: - from app.alert_pipeline import get_active_alert_count, get_recent_alerts + from app.domains.intelligence.alert_pipeline import ( + get_active_alert_count, + get_recent_alerts, + ) count = await get_active_alert_count() recent = await get_recent_alerts(limit=kwargs.get("limit", 20)) diff --git a/app/domains/databus/webhooks.py b/app/domains/databus/webhooks.py index 0574260..35d990c 100644 --- a/app/domains/databus/webhooks.py +++ b/app/domains/databus/webhooks.py @@ -152,7 +152,7 @@ async def handle_webhook(service: str, payload: dict, headers: dict, raw_body: b risk = _assess_webhook_risk(service, event_type, payload) if risk["level"] in ("HIGH", "CRITICAL"): try: - from app.alert_pipeline import push_alert + from app.domains.intelligence.alert_pipeline import push_alert await push_alert( title=f"[{service.upper()}] {risk['title']}", diff --git a/app/domains/intelligence/__init__.py b/app/domains/intelligence/__init__.py new file mode 100644 index 0000000..6123681 --- /dev/null +++ b/app/domains/intelligence/__init__.py @@ -0,0 +1,25 @@ +"""Intelligence domain - public API. + +Phase 4 domain consolidation: intelligence-related modules -> app.domains.intelligence. +""" +from __future__ import annotations + +from app.domains.intelligence.alert_pipeline import ( + get_active_alert_count, + get_recent_alerts, + push_alert, + run_alert_scan, + scan_known_scams, + scan_solana_new_pairs, +) +from app.domains.intelligence.intel_pipeline import run_intelligence_cycle + +__all__ = [ + "get_active_alert_count", + "get_recent_alerts", + "push_alert", + "run_alert_scan", + "run_intelligence_cycle", + "scan_known_scams", + "scan_solana_new_pairs", +] diff --git a/app/domains/intelligence/alert_pipeline.py b/app/domains/intelligence/alert_pipeline.py new file mode 100644 index 0000000..f93c7f9 --- /dev/null +++ b/app/domains/intelligence/alert_pipeline.py @@ -0,0 +1,370 @@ +""" +RMI Alert Pipeline - Real-time threat intelligence from scanners. +================================================================= +Feeds the Live Intel panel, WebSocket streams, and alert endpoints. + +Data sources: + - SENTINEL scanner (risk scores, scam detection) + - GoPlus security API (honeypot, tax, proxy checks) + - DexScreener new pairs (fresh launches to scan) + - Our own RAG scam patterns + - Wallet label cross-references + +Alert flow: + Scanner → alert_pipeline.push_alert() → Redis sorted set + pub/sub + Homepage reads from /api/v1/alerts/recent + Sidebar reads from /api/v1/alerts/count + WebSocket streams to connected clients +""" + +import json +import logging +import os +import time +from datetime import UTC, datetime + +logger = logging.getLogger("alert_pipeline") + +REDIS_HOST = os.getenv("REDIS_HOST", "rmi-redis") +REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) +REDIS_PW = os.getenv("REDIS_PASSWORD", "") + +ALERT_KEY = "rmi:alerts:recent" +ALERT_COUNT_KEY = "rmi:alerts:count:active" +ALERT_MAX = 500 # Keep last 500 alerts + + +async def _get_redis(): + """Get async Redis connection.""" + import redis.asyncio as aioredis + + return aioredis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PW or None, + decode_responses=True, + ) + + +async def get_active_alert_count() -> int: + """Get count of active (unacknowledged) alerts.""" + try: + r = await _get_redis() + count = await r.zcard(ALERT_KEY) + await r.close() + return count + except Exception as e: + logger.debug(f"Alert count error: {e}") + return 0 + + +async def push_alert( + alert_type: str, + title: str, + description: str = "", + severity: str = "high", + chain: str = "unknown", + token: str = "", + token_symbol: str = "", + wallet: str = "", + risk_score: int = 0, + metadata: dict | None = None, +) -> str: + """ + Push a new alert to the pipeline. + Returns alert_id. + """ + alert_id = f"alt_{int(time.time())}_{os.urandom(4).hex()}" + + alert = { + "id": alert_id, + "type": alert_type, + "title": title, + "description": description, + "severity": severity, + "chain": chain, + "token": token, + "token_symbol": token_symbol, + "wallet": wallet, + "risk_score": risk_score, + "acknowledged": False, + "created_at": datetime.now(UTC).isoformat(), + **(metadata or {}), + } + + try: + r = await _get_redis() + score = time.time() + await r.zadd(ALERT_KEY, {json.dumps(alert): score}) + # Trim old alerts + await r.zremrangebyrank(ALERT_KEY, 0, -(ALERT_MAX + 1)) + # Pub/sub for WebSocket streaming + await r.publish("rmi:ws:alerts", json.dumps(alert)) + await r.close() + logger.info(f"Alert pushed: {alert_type} | {title[:60]}") + except Exception as e: + logger.error(f"Failed to push alert: {e}") + + return alert_id + + +async def get_recent_alerts(limit: int = 20, severity: str = "") -> list[dict]: + """Get recent alerts with optional severity filter. Normalizes old/new formats.""" + try: + r = await _get_redis() + raw = await r.zrevrange(ALERT_KEY, 0, limit * 2 - 1) + await r.close() + + alerts = [] + for entry in raw: + try: + alert = json.loads(entry) + + # Normalize old alert format to new + if "title" not in alert: + alert["title"] = alert.get("message", alert.get("event", "Unknown alert")) + if "description" not in alert: + desc_parts = [] + if alert.get("symbol"): + desc_parts.append(f"Token: {alert['symbol']}") + if alert.get("risk_score"): + desc_parts.append(f"Risk: {alert['risk_score']}/100") + flags = alert.get("risk_flags", []) + if flags: + desc_parts.append("; ".join(str(f)[:80] for f in flags[:2])) + alert["description"] = " | ".join(desc_parts) + if "severity" not in alert: + score = alert.get("risk_score", 50) + alert["severity"] = "critical" if score >= 85 else "high" if score >= 65 else "medium" + if "chain" not in alert: + alert["chain"] = alert.get("chain", "unknown") + + if severity and alert.get("severity") != severity: + continue + alerts.append(alert) + if len(alerts) >= limit: + break + except json.JSONDecodeError: + pass + + return alerts + except Exception as e: + logger.error(f"get_recent_alerts error: {e}") + return [] + + +# ── Alert generators - called by cron jobs or on-demand ────────── + + +async def scan_solana_new_pairs(limit: int = 5) -> int: + """ + Scan latest Solana pairs from DexScreener for scam patterns. + Pushes alerts for high-risk tokens. + Returns number of alerts generated. + """ + import httpx + + pushed = 0 + + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.get("https://api.dexscreener.com/latest/dex/search", params={"q": "SOL USDC"}) + if resp.status_code != 200: + return 0 + + data = resp.json() + pairs = data.get("pairs", [])[:limit] + + for pair in pairs: + token_addr = pair.get("baseToken", {}).get("address", "") + token_name = pair.get("baseToken", {}).get("name", "Unknown") + token_symbol = pair.get("baseToken", {}).get("symbol", "???") + liquidity = float(pair.get("liquidity", {}).get("usd", 0) or 0) + volume = float(pair.get("volume", {}).get("h24", 0) or 0) + price_change = float(pair.get("priceChange", {}).get("h24", 0) or 0) + created_at = pair.get("pairCreatedAt", 0) + age_hours = (time.time() - created_at / 1000) / 3600 if created_at else 999 + + # Risk heuristics + risk_flags = [] + + if liquidity < 1000: + risk_flags.append("low_liquidity") + if volume == 0: + risk_flags.append("no_volume") + if price_change < -80: + risk_flags.append(f"crashed_{abs(price_change):.0f}%") + if age_hours < 1 and liquidity < 5000: + risk_flags.append("fresh_launch_low_liq") + if price_change > 500: + risk_flags.append(f"pumped_{price_change:.0f}%") + + if risk_flags: + await push_alert( + alert_type="new_pair_risk", + title=f"{token_symbol}: {' | '.join(risk_flags[:2])}", + description=f"New pair {token_name} ({token_symbol}) on Solana. " + f"Liquidity: ${liquidity:,.0f}, Age: {age_hours:.1f}h, " + f"24h change: {price_change:+.1f}%", + severity="critical" if len(risk_flags) >= 3 else "high", + chain="solana", + token=token_addr, + token_symbol=token_symbol, + risk_score=min(90, len(risk_flags) * 25), + metadata={ + "risk_flags": risk_flags, + "liquidity_usd": liquidity, + "age_hours": age_hours, + }, + ) + pushed += 1 + + return pushed + except Exception as e: + logger.warning(f"Solana scan error: {e}") + return pushed + + +async def scan_known_scams(limit: int = 3) -> int: + """ + Check our RAG known_scams collection for recently added entries. + Pushes alerts for new scam patterns detected. + """ + pushed = 0 + try: + r = await _get_redis() + # Check for recent scam pattern additions + scam_ids = await r.smembers("rag:idx:known_scams") + + recent_count = 0 + for sid in list(scam_ids)[:20]: + doc = await r.get(f"rag:known_scams:{sid}") + if doc: + try: + data = json.loads(doc) + added = data.get("metadata", {}).get("added_at", "") + if added: + age_h = (time.time() - datetime.fromisoformat(added.replace("Z", "+00:00")).timestamp()) / 3600 + if age_h < 24: + recent_count += 1 + except Exception: + pass + + await r.close() + + if recent_count > 0: + await push_alert( + alert_type="scam_pattern_update", + title=f"{recent_count} new scam patterns detected in last 24h", + description="New rug pull, honeypot, or phishing patterns added to the knowledge base.", + severity="high", + chain="multi", + risk_score=85, + ) + pushed += 1 + + return pushed + except Exception as e: + logger.warning(f"Known scams scan error: {e}") + return pushed + + +async def run_alert_scan() -> dict[str, int]: + """ + Run a full alert scan across all sources. + Called by cron job every 15 minutes. + """ + results = {} + + # Scan Solana new pairs + results["solana_pairs"] = await scan_solana_new_pairs(limit=8) + + # Scan known scams + results["known_scams"] = await scan_known_scams() + + total = sum(results.values()) + logger.info(f"Alert scan complete: {total} new alerts ({results})") + return results + + +# ── Seed some initial alerts if Redis is empty ──────────────────── + + +async def seed_initial_alerts(): + """Seed baseline alerts so the system isn't empty on first run.""" + r = await _get_redis() + existing = await r.zcard(ALERT_KEY) + await r.close() + + if existing > 0: + return # Already has alerts + + seeds = [ + ( + "honeypot", + "Honeypot detected on Base: 0xdead...", + "Token has sell restrictions and blacklist. Buyers cannot exit.", + "critical", + "base", + ), + ( + "whale", + "Whale moved 5M USDC to Binance", + "Wallet 0xABCD... transferred $5M USDC to Binance hot wallet. Possible sell pressure.", + "high", + "ethereum", + ), + ( + "rugpull", + "Liquidity removed from $SCAM token", + "100% of liquidity pool withdrawn by deployer. Token is now worthless.", + "critical", + "solana", + ), + ( + "bundler", + "Sniper bundle detected: $NEWLAUNCH", + "Coordinated wallet cluster bought 60% of supply in first block.", + "high", + "solana", + ), + ( + "contract", + "Unverified proxy contract found", + "Token uses upgradeable proxy with unverified implementation. Owner can change logic.", + "high", + "base", + ), + ( + "concentration", + "90% supply held by 3 wallets on $DANGER", + "Extreme holder concentration. Classic rug pull setup.", + "critical", + "ethereum", + ), + ( + "wash_trade", + "Wash trading detected on $FAKEVOL", + "95% of volume is self-trading between linked wallets.", + "high", + "bsc", + ), + ( + "phishing", + "Fake airdrop targeting $BONK holders", + "Phishing site detected posing as official BONK airdrop. Users losing funds.", + "critical", + "solana", + ), + ] + + for alert_type, title, desc, severity, chain in seeds: + await push_alert( + alert_type=alert_type, + title=title, + description=desc, + severity=severity, + chain=chain, + ) + + logger.info(f"Seeded {len(seeds)} initial alerts") diff --git a/app/domains/intelligence/intel_pipeline.py b/app/domains/intelligence/intel_pipeline.py new file mode 100644 index 0000000..ff2c30f --- /dev/null +++ b/app/domains/intelligence/intel_pipeline.py @@ -0,0 +1,298 @@ +""" +RMI Intelligence Pipeline - HF + Supabase + RAG +================================================ +Unified intelligence service: scam classification (HF models), +wallet labeling via RAG pattern matching, Supabase hybrid storage. +""" + +import hashlib +import logging +import os +from datetime import UTC, datetime + +import httpx +from dotenv import load_dotenv + +load_dotenv("/app/.env", override=True) + +logger = logging.getLogger(__name__) + +HF_TOKEN = os.getenv("HF_TOKEN", "") +HF_API = "https://api-inference.huggingface.co/models" + + +def _get_url(): + return os.getenv("SUPABASE_URL", "") + + +def _get_key(): + return os.getenv("SUPABASE_SERVICE_KEY", "") + + +def _get_headers(): + key = _get_key() + return { + "apikey": key, + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + + +# ── HF Models (Paywalled - using local fallback) ────── +# HF Inference API now requires PRO subscription ($9/mo). +# Using local pattern matching + RAG memory instead. +# Enable HF by setting HUGGINGFACE_TOKEN to a valid PRO key. + +SCAM_LABELS = ["scam", "rugpull", "honeypot", "legitimate", "phishing", "ponzi"] + +SCAM_KEYWORDS = { + "scam": ["scam", "fraud", "stole", "stolen", "exit scam", "fake"], + "rugpull": ["rug", "rugpull", "pulled liquidity", "drained", "removed liquidity", "lp removed"], + "honeypot": [ + "honeypot", + "cannot sell", + "can't sell", + "unable to sell", + "transfer disabled", + "sell tax 100", + ], + "phishing": [ + "phishing", + "airdrop scam", + "claim reward", + "verify wallet", + "seed phrase", + "private key", + ], + "ponzi": [ + "ponzi", + "mlm", + "multi level", + "referral rewards", + "guaranteed returns", + "double your", + ], + "insider": ["insider", "team wallet", "dev wallet", "pre-sale", "unlocked tokens", "vesting"], +} + + +async def classify_scam_risk(text: str) -> dict: + """Classify scam risk using local pattern matching (HF paywalled).""" + text_lower = text.lower() + scores = {} + + for label, keywords in SCAM_KEYWORDS.items(): + score = sum(1 for kw in keywords if kw in text_lower) + if score > 0: + scores[label] = min(score / len(keywords), 1.0) + + try: + from app.rag_service import search_documents + + rag_results = await search_documents("known_scams", text, limit=3) + for r in rag_results: + content = r.get("content", "").lower() + for label, keywords in SCAM_KEYWORDS.items(): + if any(kw in content for kw in keywords): + scores[label] = scores.get(label, 0) + 0.1 + except Exception: + pass + + if not scores: + return { + "risk": "low", + "labels": {}, + "confidence": 0, + "is_scam": False, + "risk_level": "low", + "model": "local_pattern_match", + } + + top = max(scores, key=scores.get) + top_score = scores[top] + is_scam = top in ("scam", "rugpull", "honeypot", "phishing", "ponzi") + + return { + "model": "local_pattern_match", + "labels": {k: round(v, 3) for k, v in sorted(scores.items(), key=lambda x: x[1], reverse=True)}, + "top_label": top, + "confidence": round(top_score, 3), + "is_scam": is_scam, + "risk_level": "high" if (is_scam and top_score > 0.5) else "medium" if is_scam else "low", + } + + +async def generate_embedding(text: str) -> list[float] | None: + """Generate embedding vector for RAG storage using HF free tier.""" + if not HF_TOKEN: + return None + + async with httpx.AsyncClient(timeout=60) as client: + r = await client.post( + f"{HF_API}/{EMBEDDING_MODEL}", # noqa: F821 -- pre-existing bug, see fix(f821) tracking issue + headers={"Authorization": f"Bearer {HF_TOKEN}"}, + json={"inputs": text[:1024]}, + ) + if r.status_code == 503: + logger.warning("HF embedding cold start, model loading") + return None + if r.status_code == 200: + result = r.json() + # Handle different response formats + if isinstance(result, list) and len(result) > 0: + return result[0] if isinstance(result[0], list) else result + return result + logger.warning(f"HF embedding failed: {r.status_code}") + return None + + +# ── Wallet Labeling via Pattern Memory ────────────────── + +WALLET_PATTERNS = { + "sybil_farmer": [ + "funded from exchange within seconds of 100+ other wallets", + "identical funding amounts across multiple wallets", + "no organic activity, only test transactions", + "funded by known Sybil distributor", + ], + "wash_trader": [ + "circular transfers between related wallets", + "buys and sells same token within minutes", + "volume spikes without holder count changes", + "coordinated buy/sell patterns", + ], + "sandwich_bot": [ + "high frequency trading with frontrun pattern", + "buys before large buys, sells immediately after", + "MEV extraction patterns", + "flashbots bundle usage", + ], + "liquidity_remover": [ + "large LP removal shortly after token launch", + "multiple LP positions removed simultaneously", + "liquidity drained to fresh wallet", + "LP removal preceded by marketing push", + ], + "honeypot_deployer": [ + "deploys tokens that can't be sold", + "reuses contract code across multiple tokens", + "disables transfers after liquidity added", + "ownership not renounced, hidden mint functions", + ], + "phishing_operator": [ + "sends tokens to many addresses with scam links", + "impersonates legitimate token contracts", + "uses airdrop as phishing vector", + "connects to known phishing domains", + ], + "mixer_user": [ + "funds pass through Tornado Cash or similar", + "receives from mixer, sends to clean wallet", + "layered mixing through multiple hops", + "funds originate from high-risk sources", + ], + "insider_trader": [ + "buys tokens before public announcements", + "linked to team wallets or deployers", + "sells immediately after hype peak", + "coordinated timing with other insiders", + ], +} + + +async def label_wallet(wallet_data: dict) -> dict: + """Label a wallet based on behavioral patterns and RAG memory.""" + labels = [] + confidence_scores = {} + + # Check behavioral patterns + behavior = wallet_data.get("behavior_summary", "") + wallet_data.get("transactions", []) + + for label, patterns in WALLET_PATTERNS.items(): + score = 0 + for pattern in patterns: + if pattern.lower() in behavior.lower(): + score += 1 + if score > 0: + confidence = min(score / len(patterns), 1.0) + confidence_scores[label] = round(confidence, 2) + if confidence > 0.3: + labels.append({"label": label, "confidence": round(confidence, 2)}) + + # Query RAG for similar wallet patterns + try: + from app.rag_service import search_documents + + rag_results = await search_documents( + "wallet_profiles", wallet_data.get("address", "") + " " + behavior, limit=5 + ) + if rag_results: + for r in rag_results: + content = r.get("content", "") + for label, patterns in WALLET_PATTERNS.items(): + if any(p.lower() in content.lower() for p in patterns): + if label not in confidence_scores: + confidence_scores[label] = 0 + confidence_scores[label] = min(confidence_scores[label] + 0.15, 1.0) + except Exception: + pass + + labels = [ + {"label": k, "confidence": v} + for k, v in sorted(confidence_scores.items(), key=lambda x: x[1], reverse=True) + if v > 0.2 + ] + + return { + "wallet_address": wallet_data.get("address", "unknown"), + "labels": labels[:5], + "primary_label": labels[0]["label"] if labels else "unknown", + "risk_score": max([line["confidence"] for line in labels]) * 100 if labels else 0, + "analyzed_at": datetime.now(UTC).isoformat(), + } + + +# ── Supabase Hybrid Storage ──────────────────────────── + + +async def sync_to_supabase(collection: str, document: dict) -> dict: + """Sync RAG document to Supabase for persistent hybrid storage.""" + if not _get_url() or not _get_key(): + return {"status": "skipped", "reason": "No Supabase config"} + + doc_id = hashlib.sha256(f"{collection}:{document.get('content', '')[:100]}".encode()).hexdigest()[:16] + + payload = { + "document_id": doc_id, + "collection": collection, + "content": document.get("content", "")[:5000], + "metadata": document.get("metadata", {}), + "synced_at": datetime.now(UTC).isoformat(), + } + + headers = _get_headers() + headers["Prefer"] = "resolution=merge-duplicates" + + async with httpx.AsyncClient(timeout=15) as client: + r = await client.post(f"{_get_url()}/rest/v1/rag_documents", json=payload, headers=headers) + if r.status_code in (200, 201, 409): + return {"status": "synced", "doc_id": doc_id, "supabase_status": r.status_code} + return {"status": "failed", "error": r.text[:200]} + + +# ── Batch Processing ─────────────────────────────────── + + +async def run_intelligence_cycle() -> dict: + """Run a full intelligence cycle: classify, label, embed, sync.""" + results = { + "cycle": datetime.now(UTC).isoformat(), + "scam_checks": 0, + "wallet_labels": 0, + "embeddings": 0, + "supabase_syncs": 0, + "errors": [], + } + + return results diff --git a/app/domains/intelligence/meme_intelligence.py b/app/domains/intelligence/meme_intelligence.py new file mode 100644 index 0000000..7201df4 --- /dev/null +++ b/app/domains/intelligence/meme_intelligence.py @@ -0,0 +1,423 @@ +""" +Meme Intelligence Platform - Meme Coin Tracking, Smart Money in Memes, +Big Wins/Losses, KOL Scorecards, Social Monitoring. + +Integrations: +- DexScreener: meme token launches, trending +- LunarCrush: social sentiment, social dominance +- X/Twitter: KOL posts, viral content +- Telegram: channel monitoring, group sentiment +- Arkham: whale tracking in memes +- Birdeye: Solana meme tokens +- Pump.fun: new meme launches +""" + +import logging +import os + +from dotenv import load_dotenv + +load_dotenv("/app/.env", override=True) +from datetime import UTC, datetime, timedelta # noqa: E402 + +logger = logging.getLogger(__name__) + +# ── Meme Intelligence Data Structures ───────────────────────── + +MEME_CHAINS = ["solana", "ethereum", "base", "bsc", "arbitrum"] + +MEME_CATEGORIES = { + "dog": ["dog", "doge", "shib", "akita", "kishu"], + "cat": ["cat", "pepe", "mog", "meow"], + "politi": ["trump", "biden", "maga", "politics"], + "celeb": ["celeb", "influencer", "famous"], + "ai": ["ai", "gpt", "neural", "chat"], + "gaming": ["game", "gaming", "nft", "metaverse"], + "other": [], +} + +# KOL Database Schema +KOL_DATABASE = { + # Example structure - populate from research + "twitter_handles": [], + "telegram_channels": [], + "wallet_addresses": [], + "track_record": {}, # past calls, win rate + "follower_counts": {}, + "engagement_rates": {}, +} + +# ── Meme Token Intelligence ──────────────────────────────────── + + +async def get_meme_trending(limit: int = 50) -> list[dict]: + """Get trending meme tokens across chains.""" + from app.unified_provider import get_unified_provider + + provider = get_unified_provider() + memes = [] + + for chain in MEME_CHAINS: + try: + # Get trending from DexScreener + trending = await provider.get_dexscreener_trending(chain) + + for token in trending[:20]: + # Classify as meme based on name/symbol + category = classify_meme_category( + token.get("baseToken", {}).get("symbol", ""), + token.get("baseToken", {}).get("name", ""), + ) + + if category != "other": + memes.append( + { + "address": token.get("baseToken", {}).get("address"), + "symbol": token.get("baseToken", {}).get("symbol"), + "name": token.get("baseToken", {}).get("name"), + "chain": chain, + "category": category, + "price_usd": token.get("priceUsd"), + "volume_24h": token.get("volume", {}).get("h24"), + "price_change_24h": token.get("priceChange", {}).get("h24"), + "liquidity_usd": token.get("liquidity", {}).get("usd"), + "fdv": token.get("fdv"), + "pair_age_hours": token.get("pairCreatedAt"), + "is_meme": True, + } + ) + except Exception as e: + logger.debug(f"Error getting meme trending for {chain}: {e}") + + # Sort by volume + memes.sort(key=lambda x: x.get("volume_24h", 0), reverse=True) + + return memes[:limit] + + +async def get_smart_money_in_memes(limit: int = 20) -> list[dict]: + """Track known smart money wallets trading memes.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Query for smart money activities in meme tokens + result = ( + supabase.table("smart_money_activities") + .select(""" + *, + wallets ( + wallet_address, + wallet_label, + wallet_category, + win_rate, + total_pnl_usd + ) + """) + .eq("is_meme", True) + .order("amount_usd", desc=True) + .limit(limit) + .execute() + ) + + return result.data or [] + + +async def get_meme_wins_losses(period: str = "24h") -> dict: + """Get biggest wins and losses in meme tokens.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Calculate time range + if period == "24h": + time_range = datetime.now(UTC) - timedelta(hours=24) + elif period == "7d": + time_range = datetime.now(UTC) - timedelta(days=7) + elif period == "30d": + time_range = datetime.now(UTC) - timedelta(days=30) + else: + time_range = datetime.now(UTC) - timedelta(hours=24) + + # Get biggest wins + wins = ( + supabase.table("whale_movements") + .select("*") + .gte("collected_at", time_range.isoformat()) + .eq("transaction_type", "sell") + .order("amount_usd", desc=True) + .limit(20) + .execute() + ) + + # Get biggest losses + losses = ( + supabase.table("whale_movements") + .select("*") + .gte("collected_at", time_range.isoformat()) + .eq("transaction_type", "buy") + .order("amount_usd", desc=True) + .limit(20) + .execute() + ) + + return { + "period": period, + "wins": wins.data or [], + "losses": losses.data or [], + } + + +def classify_meme_category(symbol: str, name: str) -> str: + """Classify meme token into category.""" + text = f"{symbol} {name}".lower() + + for category, keywords in MEME_CATEGORIES.items(): + if any(kw in text for kw in keywords): + return category + + return "other" + + +# ── KOL Intelligence ────────────────────────────────────────── + + +async def get_kol_scorecard(kol_handle: str) -> dict | None: + """Get KOL scorecard with track record.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Get KOL profile + result = supabase.table("kols").select("*").eq("twitter_handle", kol_handle).execute() + + if not result.data: + return None + + kol = result.data[0] + + # Get their past calls + calls = ( + supabase.table("kol_calls") + .select("*") + .eq("kol_id", kol["id"]) + .order("called_at", desc=True) + .limit(50) + .execute() + ) + + # Calculate stats + total_calls = len(calls.data) if calls.data else 0 + winning_calls = len([c for c in (calls.data or []) if c.get("pnl_pct", 0) > 0]) + win_rate = (winning_calls / total_calls * 100) if total_calls > 0 else 0 + + avg_pnl = sum([c.get("pnl_pct", 0) for c in (calls.data or [])]) / total_calls if total_calls > 0 else 0 + + return { + "kol": kol, + "stats": { + "total_calls": total_calls, + "winning_calls": winning_calls, + "win_rate": round(win_rate, 2), + "average_pnl": round(avg_pnl, 2), + "follower_count": kol.get("follower_count"), + "engagement_rate": kol.get("engagement_rate"), + }, + "recent_calls": calls.data[:10] if calls.data else [], + } + + +async def get_top_kols_by_category(category: str = "memes", limit: int = 20) -> list[dict]: + """Get top KOLs by category with scorecards.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + result = ( + supabase.table("kols") + .select("*") + .eq("primary_category", category) + .order("win_rate", desc=True) + .order("follower_count", desc=True) + .limit(limit) + .execute() + ) + + return result.data or [] + + +# ── Social Monitoring ───────────────────────────────────────── + + +async def monitor_social_sentiment(token_address: str) -> dict: + """Monitor social sentiment for a token.""" + # LunarCrush integration + try: + from app.lunarcrush_connector import get_lunarcrush_connector + + lc = get_lunarcrush_connector() + + sentiment = await lc.get_sentiment(token_address) + + return { + "token": token_address, + "social_volume": sentiment.get("social_volume"), + "social_dominance": sentiment.get("social_dominance"), + "sentiment_score": sentiment.get("sentiment_score"), + "mentions_24h": sentiment.get("mentions_24h"), + "mentions_change_24h": sentiment.get("mentions_change_24h"), + } + except Exception: + return {"error": "LunarCrush not available"} + + +async def get_viral_crypto_posts(hours: int = 24) -> list[dict]: + """Get viral crypto posts from X/Twitter.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + time_range = datetime.now(UTC) - timedelta(hours=hours) + + result = ( + supabase.table("viral_posts") + .select("*") + .gte("posted_at", time_range.isoformat()) + .order("engagement_score", desc=True) + .limit(50) + .execute() + ) + + return result.data or [] + + +# ── Hack/Drain Alerts ──────────────────────────────────────── + + +async def get_recent_hacks_drains(hours: int = 24) -> list[dict]: + """Get recent hacks and drains from monitoring.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + time_range = datetime.now(UTC) - timedelta(hours=hours) + + result = ( + supabase.table("security_alerts") + .select("*") + .gte("detected_at", time_range.isoformat()) + .in_("alert_type", ["hack", "drain", "exploit", "rugpull"]) + .order("amount_usd", desc=True) + .limit(50) + .execute() + ) + + return result.data or [] + + +async def format_hack_alert_for_social(hack_data: dict) -> dict: + """Format hack alert for X/Telegram posting.""" + return { + "x_post": f"""🚨 HACK ALERT 🚨 + +Protocol: {hack_data.get("protocol_name", "Unknown")} +Amount: ${hack_data.get("amount_usd", 0):,.0f} +Chain: {hack_data.get("chain", "Unknown")} +Type: {hack_data.get("attack_type", "Unknown")} + +{hack_data.get("description", "")[:200]} + +#CryptoSecurity #DeFi #HackAlert""", + "telegram_post": f"""🚨 *HACK ALERT* 🚨 + +*Protocol:* {hack_data.get("protocol_name", "Unknown")} +*Amount:* ${hack_data.get("amount_usd", 0):,.0f} +*Chain:* {hack_data.get("chain", "Unknown")} +*Type:* {hack_data.get("attack_type", "Unknown")} + +{hack_data.get("description", "")[:500]} + +Stay safe out there! 🔒""", + "severity": hack_data.get("severity", "medium"), + "amount_usd": hack_data.get("amount_usd", 0), + } + + +# ── Wallet Screenshot Generation ────────────────────────────── + + +async def generate_wallet_screenshot(wallet_address: str, pnl_data: dict) -> str: + """Generate wallet PnL screenshot for sharing.""" + # This would use a graphics API (Alibaba, etc.) to generate images + # For now, return placeholder + + return { + "wallet": wallet_address, + "total_pnl": pnl_data.get("total_pnl_usd", 0), + "win_rate": pnl_data.get("win_rate", 0), + "top_wins": pnl_data.get("top_wins", []), + "top_losses": pnl_data.get("top_losses", []), + "image_url": f"/api/v1/images/wallet/{wallet_address}/pnl-summary", + "share_url": f"https://rugmunch.io/wallet/{wallet_address}", + } + + +# ── Content Generation ──────────────────────────────────────── + + +async def generate_marketing_content(content_type: str, data: dict) -> dict: + """Generate marketing content using AI.""" + # This would call Alibaba's AI API for content generation + + templates = { + "win_announcement": """ +🎉 BIG WIN ALERT! 🎉 + +Wallet: {wallet_address[:8]}...{wallet_address[-6:]} +Token: {token_symbol} +Profit: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) + +This whale called it early and rode it all the way up! 🐋 + +Track smart money: https://rugmunch.io/wallet/{wallet_address} + +#Crypto #MemeCoin #SmartMoney +""", + "loss_announcement": """ +💀 LOSS PORN 💀 + +Wallet: {wallet_address[:8]}...{wallet_address[-6:]} +Token: {token_symbol} +Loss: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) + +Oof. Another reminder to take profits! 📉 + +Learn from their mistakes: https://rugmunch.io/wallet/{wallet_address} + +#Crypto #Trading #LossPorn +""", + "kol_scorecard": """ +📊 KOL SCORECARD: @{kol_handle} + +Win Rate: {win_rate:.1f}% +Total Calls: {total_calls} +Avg PnL: {avg_pnl:.1f}% +Followers: {follower_count:,} + +Track their calls: https://rugmunch.io/kol/{kol_handle} + +#CryptoTwitter #KOL #Alpha +""", + } + + template = templates.get(content_type, "") + + # Format template with data + content = template.format(**data) if template else "" + + return { + "content_type": content_type, + "x_post": content[:280], + "telegram_post": content, + "generated_at": datetime.now(UTC).isoformat(), + } diff --git a/app/domains/intelligence/n8n_intelligence_workflows.py b/app/domains/intelligence/n8n_intelligence_workflows.py new file mode 100644 index 0000000..763ec33 --- /dev/null +++ b/app/domains/intelligence/n8n_intelligence_workflows.py @@ -0,0 +1,438 @@ +""" +n8n Workflow Automation - Market Intelligence & Premium Data Pulls. +Scheduled workflows for efficient data collection from multiple sources. +Runs every 30-60 minutes based on scan volume and data freshness needs. + +Integrations: +- CoinGecko: trending, global metrics, top gainers/losers +- DexScreener: new pairs, trending tokens, volume spikes +- Dune Analytics: custom queries for whale tracking, scam patterns +- Helius: token mints, large transfers, new token deployments +- Moralis: EVM whale movements, new contract deployments +- Arkham: entity labeling, exchange flows +- Nansen: smart money tracking (if available) +- Forta: real-time threat detection +""" + +import logging +from datetime import UTC, datetime + +logger = logging.getLogger(__name__) + +# ── n8n Workflow Definitions ───────────────────────────────── + +N8N_BASE_URL = "https://n8n.rugmunch.io" +N8N_WEBHOOK_URL = f"{N8N_BASE_URL}/webhook" + +# Market Intelligence Workflows +MARKET_INTEL_WORKFLOWS = { + "trending_tokens": { + "name": "Trending Tokens Collector", + "schedule": "*/30 * * * *", # Every 30 minutes + "sources": ["coingecko", "dexscreener", "geckoterminal"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/trending", + "description": "Collect trending tokens from multiple DEXs", + "output_table": "market_trending_tokens", + }, + "whale_movements": { + "name": "Whale Movement Tracker", + "schedule": "*/15 * * * *", # Every 15 minutes (high priority) + "sources": ["helius", "moralis", "arkham"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/whales", + "description": "Track large transfers across chains", + "output_table": "whale_movements", + }, + "new_token_deployments": { + "name": "New Token Deployments", + "schedule": "*/10 * * * *", # Every 10 minutes (fast detection) + "sources": ["helius", "moralis", "dexscreener"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/new-tokens", + "description": "Detect new token deployments in real-time", + "output_table": "new_token_deployments", + }, + "volume_spikes": { + "name": "Volume Spike Detector", + "schedule": "*/5 * * * *", # Every 5 minutes (critical) + "sources": ["dexscreener", "geckoterminal", "birdeye"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/volume-spikes", + "description": "Detect unusual volume increases", + "output_table": "volume_spikes", + }, + "global_metrics": { + "name": "Market Global Metrics", + "schedule": "0 * * * *", # Every hour + "sources": ["coingecko"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/global", + "description": "Global crypto market metrics", + "output_table": "market_global_metrics", + }, + "defi_protocols": { + "name": "DeFi Protocol Analytics", + "schedule": "0 */2 * * *", # Every 2 hours + "sources": ["defillama", "dune"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/defi", + "description": "DeFi TVL, volume, user metrics", + "output_table": "defi_protocol_metrics", + }, + "nft_market": { + "name": "NFT Market Intelligence", + "schedule": "0 */4 * * *", # Every 4 hours + "sources": ["alchemy", "opensea_api"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/nft", + "description": "NFT floor prices, volume, trends", + "output_table": "nft_market_metrics", + }, +} + +# Premium Intelligence Workflows +PREMIUM_INTEL_WORKFLOWS = { + "smart_money_tracking": { + "name": "Smart Money Tracker", + "schedule": "*/30 * * * *", # Every 30 minutes + "sources": ["nansen", "arkham", "dune"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/smart-money", + "description": "Track known smart money wallets", + "output_table": "smart_money_activities", + "tier": "premium", + }, + "exchange_flows": { + "name": "Exchange Flow Analysis", + "schedule": "*/15 * * * *", # Every 15 minutes + "sources": ["arkham", "dune", "glassnode"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/exchange-flows", + "description": "Track exchange inflows/outflows", + "output_table": "exchange_flows", + "tier": "premium", + }, + "insider_trading": { + "name": "Insider Trading Detection", + "schedule": "*/20 * * * *", # Every 20 minutes + "sources": ["dune", "arkham", "helius"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/insider", + "description": "Detect potential insider trading patterns", + "output_table": "insider_trading_alerts", + "tier": "premium_plus", + }, + "launchpad_monitor": { + "name": "Launchpad Monitor", + "schedule": "*/10 * * * *", # Every 10 minutes + "sources": ["dexscreener", "pumpfun_api", "geckoterminal"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/launchpad", + "description": "Monitor new launches across platforms", + "output_table": "launchpad_monitoring", + "tier": "premium", + }, + "cluster_analysis": { + "name": "Cluster Pattern Analysis", + "schedule": "0 */2 * * *", # Every 2 hours + "sources": ["internal_clustering", "dune"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/clusters", + "description": "Deep cluster pattern analysis", + "output_table": "cluster_patterns", + "tier": "premium_plus", + }, +} + +# Security Intelligence Workflows +SECURITY_INTEL_WORKFLOWS = { + "scam_detection": { + "name": "Real-time Scam Detection", + "schedule": "*/5 * * * *", # Every 5 minutes (critical) + "sources": ["forta", "internal_ml", "community_reports"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/scams", + "description": "Detect new scam patterns", + "output_table": "scam_alerts", + "priority": "critical", + }, + "rugpull_detection": { + "name": "Rugpull Detection", + "schedule": "*/2 * * * *", # Every 2 minutes (ultra-critical) + "sources": ["internal_ml", "liquidity_monitor"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/rugpulls", + "description": "Detect rugpull patterns in real-time", + "output_table": "rugpull_alerts", + "priority": "critical", + }, + "contract_vulnerabilities": { + "name": "Contract Vulnerability Scanner", + "schedule": "0 * * * *", # Every hour + "sources": ["slither", "mythril", "internal_scanner"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/vulnerabilities", + "description": "Scan new contracts for vulnerabilities", + "output_table": "contract_vulnerabilities", + "priority": "high", + }, + "phishing_detection": { + "name": "Phishing Domain Detection", + "schedule": "0 */6 * * *", # Every 6 hours + "sources": ["guardian", "cryptoscamdb", "community"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/phishing", + "description": "Detect new phishing domains", + "output_table": "phishing_domains", + "priority": "high", + }, +} + + +# ── n8n Workflow Execution ──────────────────────────────────── + + +async def trigger_n8n_workflow(workflow_name: str, data: dict | None = None) -> bool: + """Trigger an n8n workflow via webhook.""" + import httpx + + # Find workflow config + all_workflows = { + **MARKET_INTEL_WORKFLOWS, + **PREMIUM_INTEL_WORKFLOWS, + **SECURITY_INTEL_WORKFLOWS, + } + workflow = all_workflows.get(workflow_name) + + if not workflow: + logger.error(f"Workflow not found: {workflow_name}") + return False + + webhook_url = workflow["endpoint"] + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post( + webhook_url, + json={ + "workflow": workflow_name, + "triggered_at": datetime.now(UTC).isoformat(), + "data": data or {}, + }, + ) + + if response.status_code in (200, 201, 202): + logger.info(f"Triggered workflow: {workflow_name}") + return True + else: + logger.error(f"Workflow trigger failed: {workflow_name} - {response.status_code}") + return False + + except Exception as e: + logger.error(f"Workflow trigger error: {workflow_name} - {e}") + return False + + +async def execute_market_intelligence_pull(workflow_name: str): + """Execute a market intelligence data pull.""" + workflow = MARKET_INTEL_WORKFLOWS.get(workflow_name) + if not workflow: + return + + logger.info(f"Executing market intel pull: {workflow['name']}") + + # Collect data from sources + collected_data = { + "workflow": workflow_name, + "sources": workflow["sources"], + "collected_at": datetime.now(UTC).isoformat(), + "data": {}, + } + + # Source-specific collection logic + for source in workflow["sources"]: + try: + if source == "coingecko": + from app.coingecko_connector import get_coingecko_connector + + connector = get_coingecko_connector() + + if "trending" in workflow_name.lower(): + trending = await connector.get_trending() + collected_data["data"]["coingecko_trending"] = trending + elif "global" in workflow_name.lower(): + global_data = await connector.get_global_metrics() + collected_data["data"]["coingecko_global"] = global_data + + elif source == "dexscreener": + from app.unified_provider import get_unified_provider + + provider = get_unified_provider() + + if "trending" in workflow_name.lower(): + trending = await provider.get_dexscreener_trending() + collected_data["data"]["dexscreener_trending"] = trending + elif "volume" in workflow_name.lower(): + spikes = await provider.get_volume_spikes() + collected_data["data"]["dexscreener_spikes"] = spikes + + elif source == "helius": + from app.chain_client import get_chain_client + + client = get_chain_client() + + if "new" in workflow_name.lower(): + new_tokens = await client.get_new_token_mints(limit=50) + collected_data["data"]["helius_new_tokens"] = new_tokens + elif "whale" in workflow_name.lower(): + whales = await client.get_large_transfers(limit=20) + collected_data["data"]["helius_whales"] = whales + + except Exception as e: + logger.error(f"Error collecting from {source}: {e}") + collected_data["data"][source] = {"error": str(e)} + + # Store in Supabase + await _store_intelligence_data(workflow["output_table"], collected_data) + + # Trigger alerts if needed + await _process_intelligence_alerts(workflow_name, collected_data) + + +async def _store_intelligence_data(table_name: str, data: dict): + """Store intelligence data in Supabase.""" + try: + import os + + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Insert data + supabase.table(table_name).insert( + { + "data": data, + "collected_at": data.get("collected_at"), + "workflow": data.get("workflow"), + } + ).execute() + + logger.info(f"Stored intelligence data in {table_name}") + + except Exception as e: + logger.error(f"Failed to store intelligence data: {e}") + + +async def _process_intelligence_alerts(workflow_name: str, data: dict): + """Process intelligence data and trigger alerts.""" + # Check for significant findings + if "whale" in workflow_name.lower(): + # Check for unusually large transfers + whales = data.get("data", {}).get("helius_whales", []) + for whale in whales: + amount = whale.get("amount", 0) + if amount > 1000: # >1000 SOL + await _create_alert("whale_movement", whale) + + elif "scam" in workflow_name.lower() or "rugpull" in workflow_name.lower(): + # Immediate alert for security issues + await _create_alert("security_critical", data) + + elif "volume" in workflow_name.lower(): + # Check for unusual volume spikes + spikes = data.get("data", {}).get("dexscreener_spikes", []) + for spike in spikes: + if spike.get("volume_change_pct", 0) > 500: # >500% increase + await _create_alert("volume_spike", spike) + + +async def _create_alert(alert_type: str, data: dict): + """Create an intelligence alert.""" + try: + import os + + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + alert_data = { + "alert_type": alert_type, + "severity": "critical" if "security" in alert_type else "high", + "data": data, + "created_at": datetime.now(UTC).isoformat(), + "is_read": False, + } + + supabase.table("intelligence_alerts").insert(alert_data).execute() + + logger.info(f"Created intelligence alert: {alert_type}") + + except Exception as e: + logger.error(f"Failed to create alert: {e}") + + +# ── Dune Analytics Integration ──────────────────────────────── + + +async def execute_dune_query(query_id: str, params: dict | None = None) -> dict | None: + """Execute a Dune Analytics query.""" + import os + + import httpx + + dune_api_key = os.getenv("DUNE_API_KEY", "") + if not dune_api_key: + logger.warning("DUNE_API_KEY not configured") + return None + + try: + async with httpx.AsyncClient(timeout=60.0) as client: + # Execute query + response = await client.post( + f"https://api.dune.com/api/v1/query/{query_id}/execute", + headers={"X-Dune-API-Key": dune_api_key}, + json=params or {}, + ) + + if response.status_code != 200: + logger.error(f"Dune query failed: {response.status_code}") + return None + + execution_id = response.json().get("execution_id") + + # Wait for results + import asyncio + + for _ in range(10): # Max 10 attempts + await asyncio.sleep(2) + + result_response = await client.get( + f"https://api.dune.com/api/v1/execution/{execution_id}/results", + headers={"X-Dune-API-Key": dune_api_key}, + ) + + if result_response.status_code == 200: + result = result_response.json() + if result.get("state") == "QUERY_STATE_COMPLETED": + return result.get("result", {}).get("rows", []) + + return None + + except Exception as e: + logger.error(f"Dune query error: {e}") + return None + + +# Pre-configured Dune queries for RMI +DUNE_QUERIES = { + "ethereum_whale_transfers": { + "query_id": "1234567", # Replace with actual query ID + "description": "Track large ETH transfers from known whale wallets", + "schedule": "*/15 * * * *", + }, + "defi_protocol_volumes": { + "query_id": "2345678", + "description": "Daily DEX volumes across major protocols", + "schedule": "0 * * * *", + }, + "nft_wash_trading": { + "query_id": "3456789", + "description": "Detect potential NFT wash trading patterns", + "schedule": "0 */6 * * *", + }, + "stablecoin_flows": { + "query_id": "4567890", + "description": "Track USDC/USDT flows to/from exchanges", + "schedule": "*/30 * * * *", + }, + "new_contract_deployments": { + "query_id": "5678901", + "description": "New contract deployments with large funding", + "schedule": "*/10 * * * *", + }, +} diff --git a/app/domains/intelligence/news_intelligence.py b/app/domains/intelligence/news_intelligence.py new file mode 100644 index 0000000..2ef6565 --- /dev/null +++ b/app/domains/intelligence/news_intelligence.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +RMI News Intelligence v3 - Industry Best +========================================= +AI-powered news pipeline: categorization, sentiment, trending, briefing. +Uses MiniMax ($20/mo flat) + Ollama Cloud. +""" + +import json +import logging +import os +import urllib.request +from collections import Counter +from datetime import UTC, datetime + +logger = logging.getLogger("rmi.news_v3") +OLLAMA_KEY = os.getenv("OLLAMA_API_KEY", os.getenv("DEEPSEEK_API_KEY", "")) +OLLAMA_URL = "https://ollama.com/v1/chat/completions" + +# Simple in-memory trending tracker +_trending_topics = Counter() +_breaking_alerts = [] + + +def analyze_article(title: str, content: str = "") -> dict: + """Full AI analysis of a news article.""" + text = f"{title} {content[:300]}" + + # Category (fast, cached) + from app.ai_pipeline_v3 import classify_news + + category = classify_news(title, content) + + # Sentiment via MiniMax (batched - 1 call per article is fine at flat rate) + sentiment = "neutral" + try: + k = os.getenv("OLLAMA_API_KEY", "") + if not k: + with open("/app/.env") as f: + for line in f: + if line.startswith("OLLAMA_API_KEY"): + k = line.strip().split("=", 1)[1] + break + if not k: + return {"category": category, "sentiment": "neutral", "is_breaking": False} + body = json.dumps( + { + "model": "deepseek-v4-flash", + "messages": [ + { + "role": "system", + "content": "Classify sentiment: BULLISH BEARISH NEUTRAL. Reply one word only.", + }, + {"role": "user", "content": text[:400]}, + ], + "max_tokens": 10, + "temperature": 0.1, + } + ).encode() + req = urllib.request.Request( + OLLAMA_URL, + data=body, + headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, + ) + resp = urllib.request.urlopen(req, timeout=8) + sentiment = json.loads(resp.read())["choices"][0]["message"]["content"].strip().upper() + if "BULL" in sentiment: + sentiment = "bullish" + elif "BEAR" in sentiment: + sentiment = "bearish" + else: + sentiment = "neutral" + except Exception as e: + logger.warning(f"Sentiment failed: {e}") + + # Track trending topics + for word in title.lower().split(): + if len(word) > 4 and word not in ( + "after", + "before", + "while", + "could", + "would", + "should", + "their", + "there", + "these", + "those", + "about", + "which", + ): + _trending_topics[word] += 1 + + # Detect breaking news + is_breaking = category == "SCAM" or any( + w in text.lower() for w in ["hacked", "exploited", "drained", "rug pulled", "emergency"] + ) + + return { + "category": category, + "sentiment": sentiment, + "is_breaking": is_breaking, + "analyzed_at": datetime.now(UTC).isoformat(), + } + + +def get_trending(limit: int = 10) -> list: + """Get trending topics from recent article analysis.""" + return [{"topic": word, "count": count} for word, count in _trending_topics.most_common(limit)] + + +def get_breaking() -> list: + """Get breaking news alerts.""" + return _breaking_alerts[-10:] + + +def daily_briefing() -> str: + """Generate an AI-powered daily news briefing using Ollama Cloud.""" + trending = get_trending(5) + topics = ", ".join(f"{t['topic']}({t['count']})" for t in trending) + + k = OLLAMA_KEY + body = json.dumps( + { + "model": "deepseek-v4-flash", + "messages": [ + { + "role": "system", + "content": "Write a 3-sentence daily crypto news briefing. Mention trending topics. Professional tone. Under 100 words.", + }, + {"role": "user", "content": f"Trending topics: {topics}"}, + ], + "max_tokens": 150, + "temperature": 0.5, + } + ).encode() + + try: + req = urllib.request.Request( + OLLAMA_URL, + data=body, + headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, + ) + resp = urllib.request.urlopen(req, timeout=15) + return json.loads(resp.read())["choices"][0]["message"]["content"].strip() + except Exception: + return f"Daily briefing: {topics} are trending in crypto news today." + + +def news_search(query: str, articles: list, limit: int = 10) -> list: + """Smart search across articles with relevance ranking.""" + results = [] + q_lower = query.lower() + for a in articles: + score = 0 + if q_lower in a.get("title", "").lower(): + score += 10 + if q_lower in a.get("content", "").lower(): + score += 5 + if q_lower in a.get("category", "").lower(): + score += 3 + if score > 0: + a["relevance"] = score + results.append(a) + return sorted(results, key=lambda x: x.get("relevance", 0), reverse=True)[:limit] diff --git a/app/domains/markets/__init__.py b/app/domains/markets/__init__.py new file mode 100644 index 0000000..eb9e0d0 --- /dev/null +++ b/app/domains/markets/__init__.py @@ -0,0 +1,19 @@ +"""Markets domain - public API. + +Phase 4 domain consolidation: app.prediction_market_service -> app.domains.markets. +""" +from __future__ import annotations + +from app.domains.markets.core import ( + PredictionDigest, + PredictionMarket, + PredictionMarketService, + get_prediction_market_service, +) + +__all__ = [ + "PredictionDigest", + "PredictionMarket", + "PredictionMarketService", + "get_prediction_market_service", +] diff --git a/app/domains/markets/core.py b/app/domains/markets/core.py new file mode 100644 index 0000000..0588baf --- /dev/null +++ b/app/domains/markets/core.py @@ -0,0 +1,1122 @@ +""" +RMI Prediction Market Intelligence Service +=========================================== +Multi-source prediction market data aggregation for crypto security intelligence. + +Data sources (all free, zero auth for read-only): + - Polymarket: Gamma API (search/discovery), CLOB API (prices/history), Data API (trades) + - Kalshi: REST API /series, /markets, /events, /orderbook (unauthenticated) + - Limitless: REST API /markets (Base chain, crypto-native) + - Manifold: REST API /markets (play-money, open-source sentiment signals) + +Open-source reference implementations (GitHub): + - homerun (braedonsaunders/homerun): Open-source prediction market platform for + Polymarket + Kalshi. Python strategies, backtesting, data sources, live trading. + - prediction-market-edge-bot: SX Bet + Polymarket aggregator with smart order routing. + - Awesome-Prediction-Market-Tools (aarora4): Curated directory of 50+ tools + including Oddpool (cross-venue aggregator), analytics dashboards, trading bots. + +Architecture: + - Direct external API calls (NEVER route through own API - anti-circular-dependency rule) + - All 4 sources queried in parallel with individual try/except + - Results normalized into unified PredictionMarket dataclass + - Redis caching: 30s TTL prices, 5min searches, 1hr digests + +Integration points: + - SENTINEL scanner: cross-reference token risk scores with market probability + - Wallet Memory Bank: entity/deployer reputation from prediction market odds + - RugMaps: visual correlation between market odds and on-chain wallet clusters + - x402 tools: expose as paid security intelligence endpoints + +Pitfalls: + - Polymarket Gamma API double-encodes outcomePrices/clobTokenIds as JSON strings + - One source timeout must not kill the entire call - individual try/except per source + - Prediction market data is probabilistic, not definitive - always cross-reference + with on-chain scanner results + - Don't poll every market every tick - use targeted search + category filters +""" + +import asyncio +import hashlib +import json +import logging +import os +from dataclasses import dataclass, field +from datetime import UTC, datetime + +import httpx + +logger = logging.getLogger("prediction_market") + +# ── API Endpoints ──────────────────────────────────────────────── + +POLYMARKET_GAMMA = "https://gamma-api.polymarket.com" +POLYMARKET_CLOB = "https://clob.polymarket.com" +POLYMARKET_DATA = "https://data-api.polymarket.com" + +KALSHI_BASE = "https://external-api.kalshi.com/trade-api/v2" + +LIMITLESS_BASE = "https://api.limitless.exchange" + +MANIFOLD_BASE = "https://api.manifold.markets/v0" + +# ── Category mappings for security relevance ──────────────────── + +SECURITY_KEYWORDS = [ + "hack", + "exploit", + "rug", + "scam", + "fraud", + "breach", + "leak", + "drain", + "phish", + "backdoor", + "vulnerability", + "zero-day", + "sanction", + "indict", + "arrest", + "freeze", + "seize", + "clampdown", + "depeg", + "insolvent", + "bankrupt", + "collapse", + "default", + "theft", + "heist", + "compromise", + "ransomware", + "malware", + "SEC", + "CFTC", + "DOJ", + "FBI", + "regulatory", + "enforcement", +] + +CRYPTO_KEYWORDS = [ + "bitcoin", + "ethereum", + "solana", + "crypto", + "defi", + "token", + "blockchain", + "web3", + "nft", + "stablecoin", + "usdt", + "usdc", + "dai", + "exchange", + "binance", + "coinbase", + "uniswap", + "aave", + "tether", + "circle", + "layer", + "L1", + "L2", + "rollup", + "bridge", + "polygon", + "arbitrum", + "optimism", + "avalanche", + "fantom", + "chainlink", + "makerdao", + "lido", + "eigenlayer", +] + +# ── Dataclasses ───────────────────────────────────────────────── + + +@dataclass +class PredictionMarket: + """Unified prediction market result across all sources.""" + + source: str # "polymarket" | "kalshi" | "limitless" | "manifold" + source_id: str # native ID from source (slug, ticker, etc.) + question: str + slug: str + probability_yes: float # 0.0-1.0 + probability_no: float # 0.0-1.0 + volume_usd: float + liquidity_usd: float = 0.0 + category: str = "" + tags: list[str] = field(default_factory=list) + tokens_mentioned: list[str] = field(default_factory=list) + is_security_relevant: bool = False + is_crypto_relevant: bool = False + url: str = "" + ends_at: str | None = None + updated_at: str = "" + + def __post_init__(self): + """Auto-classify relevance based on keywords in question.""" + q_lower = self.question.lower() + self.is_security_relevant = any(kw in q_lower for kw in SECURITY_KEYWORDS) + self.is_crypto_relevant = any(kw in q_lower for kw in CRYPTO_KEYWORDS) + + +@dataclass +class PredictionDigest: + """Daily intelligence digest of security-relevant prediction markets.""" + + generated_at: str + total_markets_searched: int + security_relevant_count: int + crypto_relevant_count: int + top_threats: list[PredictionMarket] = field(default_factory=list) + token_specific_markets: list[PredictionMarket] = field(default_factory=list) + ecosystem_risk_markets: list[PredictionMarket] = field(default_factory=list) + regulatory_markets: list[PredictionMarket] = field(default_factory=list) + + +# ── Service ───────────────────────────────────────────────────── + + +class PredictionMarketService: + """Multi-source prediction market data with parallel fetching + caching.""" + + def __init__(self, http_client: httpx.AsyncClient | None = None): + self._http = http_client or httpx.AsyncClient(timeout=15.0) + self._redis = None # Lazy init via get_redis() + + def _get_redis(self): + """Lazy Redis connection for caching. Returns None if unavailable.""" + if self._redis is not None: + return self._redis + try: + import redis.asyncio as aioredis + + self._redis = aioredis.from_url( + os.getenv("REDIS_URL", "redis://localhost:6379/0"), + decode_responses=True, + socket_connect_timeout=3, + ) + # Set to False if we couldn't actually connect + if self._redis is None: + self._redis = False + except Exception as e: + logger.warning(f"Redis unavailable, caching disabled: {e}") + self._redis = False + return self._redis if self._redis is not False else None + + # ── Public API ────────────────────────────────────────── + + async def search( + self, + query: str, + categories: list[str] | None = None, + min_volume: float = 0, + security_only: bool = False, + ) -> list[PredictionMarket]: + """Search all prediction market sources in parallel. + + Args: + query: Search term (token name, event, protocol, etc.) + categories: Optional filter by source categories + min_volume: Minimum USD volume to include + security_only: Only return security-relevant markets + """ + # Check Redis cache + cache_key = f"predmkt:search:{_cache_hash(query, categories, min_volume, security_only)}" + redis = self._get_redis() + if redis: + try: + cached = await redis.get(cache_key) + if cached: + markets_data = json.loads(cached) + return [_dict_to_market(d) for d in markets_data] + except Exception: + pass # Cache miss or Redis error - fall through to live query + + # Fire all 4 sources in parallel + results: list[list[PredictionMarket]] = await asyncio.gather( + self._search_polymarket(query), + self._search_kalshi(query), + self._search_limitless(query), + self._search_manifold(query), + return_exceptions=True, + ) + + # Flatten and handle exceptions + all_markets: list[PredictionMarket] = [] + sources = ["polymarket", "kalshi", "limitless", "manifold"] + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.warning(f"{sources[i]} search failed: {result}") + continue + if isinstance(result, list): + all_markets.extend(result) + + # Filter + if min_volume > 0: + all_markets = [m for m in all_markets if m.volume_usd >= min_volume] + if security_only: + all_markets = [m for m in all_markets if m.is_security_relevant] + + # Sort by volume descending + all_markets.sort(key=lambda m: m.volume_usd, reverse=True) + + # Cache (5 min TTL) + if redis: + try: + await redis.setex(cache_key, 300, json.dumps([_market_to_dict(m) for m in all_markets[:50]])) + except Exception as e: + logger.warning(f"Redis cache write failed: {e}") + + return all_markets + + async def token_markets(self, symbol: str) -> list[PredictionMarket]: + """Find all prediction markets mentioning a specific token symbol.""" + results = await self.search(f'"{symbol}" token crypto', security_only=False) + + # Filter to markets actually about this token (mention in question) + symbol_lower = symbol.lower() + token_markets = [m for m in results if symbol_lower in m.question.lower()] + return token_markets + + async def security_digest(self) -> PredictionDigest: + """Generate daily intelligence digest of security-relevant prediction markets. + + Queries crypto + security keywords across all sources, categorizes + results by threat type: top threats, token-specific, ecosystem risk, + regulatory. + """ + cache_key = f"predmkt:digest:{datetime.now(UTC).strftime('%Y-%m-%d')}" + redis = self._get_redis() + if redis: + try: + cached = await redis.get(cache_key) + if cached: + data = json.loads(cached) + return _dict_to_digest(data) + except Exception: + pass # Cache miss or Redis error - fall through to live query + + # Search for security-relevant crypto markets + security_queries = [ + "crypto hack exploit scam", + "defi rug pull fraud", + "exchange insolvent breach", + "stablecoin depeg collapse", + "crypto regulation SEC enforcement", + "blockchain vulnerability zero-day", + ] + + all_markets: list[PredictionMarket] = [] + searches = [self.search(q, security_only=False) for q in security_queries] + results = await asyncio.gather(*searches, return_exceptions=True) + + for result in results: + if isinstance(result, list): + all_markets.extend(result) + + # De-duplicate by question similarity + seen_questions = set() + unique_markets = [] + for m in all_markets: + q_key = m.question.lower().strip()[:80] + if q_key not in seen_questions: + seen_questions.add(q_key) + unique_markets.append(m) + + # Categorize + top_threats = [m for m in unique_markets if m.is_security_relevant and m.volume_usd > 10000] + top_threats.sort(key=lambda m: m.volume_usd, reverse=True) + + token_specific = [ + m for m in unique_markets if m.is_crypto_relevant and m.is_security_relevant and len(m.tokens_mentioned) > 0 + ] + token_specific.sort(key=lambda m: m.volume_usd, reverse=True) + + ecosystem_risk = [ + m for m in unique_markets if m.is_crypto_relevant and not m.is_security_relevant and m.volume_usd > 50000 + ] + ecosystem_risk.sort(key=lambda m: m.volume_usd, reverse=True) + + regulatory = [ + m + for m in unique_markets + if m.is_security_relevant + and any(kw in m.question.lower() for kw in ["sec", "cftc", "doj", "regulation", "sanction", "ban"]) + ] + regulatory.sort(key=lambda m: m.volume_usd, reverse=True) + + digest = PredictionDigest( + generated_at=datetime.now(UTC).isoformat(), + total_markets_searched=len(unique_markets), + security_relevant_count=len([m for m in unique_markets if m.is_security_relevant]), + crypto_relevant_count=len([m for m in unique_markets if m.is_crypto_relevant]), + top_threats=top_threats[:20], + token_specific_markets=token_specific[:20], + ecosystem_risk_markets=ecosystem_risk[:10], + regulatory_markets=regulatory[:10], + ) + + # Cache (1 hour) + if redis: + try: + await redis.setex(cache_key, 3600, json.dumps(_digest_to_dict(digest))) + except Exception as e: + logger.warning(f"Redis digest cache write failed: {e}") + + return digest + + async def trending(self, limit: int = 20, source: str | None = None) -> list[PredictionMarket]: + """Get top trending prediction markets by volume across all sources.""" + # Fetch top events from each source in parallel + tasks = [] + if not source or source == "polymarket": + tasks.append(self._trending_polymarket(limit)) + else: + tasks.append(asyncio.sleep(0)) # placeholder + + if not source or source == "kalshi": + tasks.append(self._trending_kalshi(limit)) + else: + tasks.append(asyncio.sleep(0)) + + if not source or source == "limitless": + tasks.append(self._trending_limitless(limit)) + else: + tasks.append(asyncio.sleep(0)) + + results = await asyncio.gather(*tasks, return_exceptions=True) + + all_markets = [] + for result in results: + if isinstance(result, list): + all_markets.extend(result) + elif isinstance(result, Exception): + pass # Individual source failures logged in _trending_* methods + + all_markets.sort(key=lambda m: m.volume_usd, reverse=True) + return all_markets[:limit] + + async def market_detail(self, source: str, market_id: str) -> PredictionMarket | None: + """Get detailed data for a specific market including orderbook.""" + if source == "polymarket": + return await self._polymarket_detail(market_id) + elif source == "kalshi": + return await self._kalshi_detail(market_id) + # Limitless and Manifold details on demand + return None + + # ── Polymarket ────────────────────────────────────────── + + async def _search_polymarket(self, query: str) -> list[PredictionMarket]: + """Search Polymarket Gamma API.""" + try: + resp = await self._http.get( + f"{POLYMARKET_GAMMA}/public-search", + params={"q": query}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Polymarket search returned {resp.status_code}") + return [] + + data = resp.json() + events = data.get("events", []) + markets = [] + + for event in events[:10]: + for m in event.get("markets", [])[:5]: + pm = self._parse_polymarket_market(m, event) + if pm: + markets.append(pm) + + return markets + except Exception as e: + logger.warning(f"Polymarket search error: {e}") + return [] + + def _parse_polymarket_market(self, m: dict, event: dict | None = None) -> PredictionMarket | None: + """Parse a Polymarket market dict into unified PredictionMarket.""" + try: + question = m.get("question", "") + slug = m.get("slug", "") + + # Parse double-encoded JSON fields + prices = self._parse_json_field(m.get("outcomePrices", "[]")) + self._parse_json_field(m.get("outcomes", "[]")) + self._parse_json_field(m.get("clobTokenIds", "[]")) + + if isinstance(prices, list) and len(prices) >= 2: + prob_yes = float(prices[0]) + prob_no = float(prices[1]) + else: + prob_yes = 0.5 + prob_no = 0.5 + + volume = float(m.get("volume", 0)) + liquidity = float(m.get("liquidity", 0)) + + # Extract token mentions from question + tokens_mentioned = _extract_token_symbols(question) + + tags = [] + if event: + tags.extend([t.get("label", "") for t in event.get("tags", [])]) + + return PredictionMarket( + source="polymarket", + source_id=slug, + question=question, + slug=slug, + probability_yes=prob_yes, + probability_no=prob_no, + volume_usd=volume, + liquidity_usd=liquidity, + category=m.get("category", event.get("category", "") if event else ""), + tags=tags, + tokens_mentioned=tokens_mentioned, + url=f"https://polymarket.com/event/{slug}" if slug else "", + ends_at=m.get("endDate", event.get("endDate", "") if event else ""), + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Polymarket market: {e}") + return None + + async def _trending_polymarket(self, limit: int) -> list[PredictionMarket]: + """Get trending Polymarket events by volume.""" + try: + resp = await self._http.get( + f"{POLYMARKET_GAMMA}/events", + params={ + "limit": limit, + "active": "true", + "closed": "false", + "order": "volume", + "ascending": "false", + }, + timeout=10.0, + ) + if resp.status_code != 200: + return [] + + events = resp.json() + markets = [] + for event in events[:limit]: + for m in event.get("markets", [])[:3]: + pm = self._parse_polymarket_market(m, event) + if pm: + markets.append(pm) + return markets + except Exception as e: + logger.warning(f"Polymarket trending error: {e}") + return [] + + async def _polymarket_detail(self, slug: str) -> PredictionMarket | None: + """Get detailed Polymarket market data including CLOB prices.""" + try: + # Fetch from Gamma + resp = await self._http.get( + f"{POLYMARKET_GAMMA}/markets", + params={"slug": slug}, + timeout=10.0, + ) + if resp.status_code != 200: + return None + + data = resp.json() + if not data: + return None + + m = data[0] + pm = self._parse_polymarket_market(m) + + # Also fetch CLOB price for live data + if pm: + tokens = self._parse_json_field(m.get("clobTokenIds", "[]")) + if isinstance(tokens, list) and len(tokens) >= 2: + try: + price_resp = await self._http.get( + f"{POLYMARKET_CLOB}/price", + params={"token_id": tokens[0], "side": "buy"}, + timeout=5.0, + ) + if price_resp.status_code == 200: + price_data = price_resp.json() + live_price = float(price_data.get("price", pm.probability_yes)) + pm.probability_yes = live_price + pm.probability_no = 1.0 - live_price + except Exception: + pass # CLOB price is a bonus, Gamma price is fine + + return pm + except Exception as e: + logger.warning(f"Polymarket detail error for {slug}: {e}") + return None + + # ── Kalshi ────────────────────────────────────────────── + + async def _search_kalshi(self, query: str) -> list[PredictionMarket]: + """Search Kalshi by scanning events then fetching their markets.""" + try: + # Step 1: Get open events (organized by category, not sports-dominant) + resp = await self._http.get( + f"{KALSHI_BASE}/events", + params={"status": "open", "limit": 50}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Kalshi events returned {resp.status_code}") + return [] + + data = resp.json() + events = data.get("events", []) + query_lower = query.lower() + query_terms = query_lower.split() + + results = [] + + # Step 2: Check event titles for matches, then fetch markets + for i, event in enumerate(events[:10]): + event_title = event.get("title", "").lower() + event_ticker = event.get("ticker", "") + + # Match if query terms appear in event title + if not any(term in event_title for term in query_terms): + continue + + # Rate limit: small delay between event fetches + if i > 0: + await asyncio.sleep(0.3) + + # Step 3: Fetch markets for this event + try: + mr = await self._http.get( + f"{KALSHI_BASE}/markets", + params={"event_ticker": event_ticker, "status": "open", "limit": 10}, + headers={"Accept": "application/json"}, + timeout=8.0, + ) + if mr.status_code == 200: + markets_data = mr.json() + for m in markets_data.get("markets", []): + pm = self._parse_kalshi_market(m) + if pm: + results.append(pm) + except Exception: + continue + + return results + except Exception as e: + logger.warning(f"Kalshi search error: {e}") + return [] + + def _parse_kalshi_market(self, m: dict) -> PredictionMarket | None: + """Parse a Kalshi market dict into unified PredictionMarket.""" + try: + ticker = m.get("ticker", "") + title = m.get("title", "") + yes_bid = float(m.get("yes_bid_dollars", 0)) + volume = float(m.get("volume_fp", 0)) # Kalshi uses fake-penny notation + m.get("event_ticker", "") + category = m.get("category", "") + + # Skip multi-outcome markets (sports parlays, etc.) - they have no yes_bid + if yes_bid <= 0 or ",yes " in title.lower(): + return None + + prob_yes = yes_bid # Best YES bid approximates probability + prob_no = 1.0 - prob_yes if prob_yes else 0.5 + + tokens_mentioned = _extract_token_symbols(title) + + return PredictionMarket( + source="kalshi", + source_id=ticker, + question=title, + slug=ticker, + probability_yes=prob_yes, + probability_no=prob_no, + volume_usd=volume, + category=category, + tokens_mentioned=tokens_mentioned, + url=f"https://kalshi.com/markets/{ticker}" if ticker else "", + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Kalshi market: {e}") + return None + + async def _trending_kalshi(self, limit: int) -> list[PredictionMarket]: + """Get trending Kalshi markets by volume - uses events-first approach.""" + try: + # Get open events (avoid sports-multi-outcome noise from raw /markets) + resp = await self._http.get( + f"{KALSHI_BASE}/events", + params={"status": "open", "limit": min(limit * 2, 30)}, + timeout=10.0, + ) + if resp.status_code != 200: + return [] + + data = resp.json() + events = data.get("events", []) + + results = [] + for event in events[:limit]: + event_ticker = event.get("ticker", "") + try: + mr = await self._http.get( + f"{KALSHI_BASE}/markets", + params={"event_ticker": event_ticker, "status": "open", "limit": 5}, + timeout=8.0, + ) + if mr.status_code == 200: + markets_data = mr.json() + for m in markets_data.get("markets", []): + pm = self._parse_kalshi_market(m) + if pm: + results.append(pm) + except Exception: + continue + + return results[:limit] + except Exception as e: + logger.warning(f"Kalshi trending error: {e}") + return [] + + async def _kalshi_detail(self, ticker: str) -> PredictionMarket | None: + """Get detailed Kalshi market data including orderbook.""" + try: + resp = await self._http.get( + f"{KALSHI_BASE}/markets/{ticker}/orderbook", + timeout=10.0, + ) + if resp.status_code != 200: + return None + + data = resp.json() + orderbook = data.get("orderbook_fp", {}) + yes_bids = orderbook.get("yes_dollars", []) + best_yes = float(yes_bids[0][0]) if yes_bids else 0.5 + + # Also get market metadata + meta_resp = await self._http.get( + f"{KALSHI_BASE}/markets", + params={"ticker": ticker}, + timeout=10.0, + ) + if meta_resp.status_code == 200: + meta_data = meta_resp.json() + markets = meta_data.get("markets", []) + if markets: + pm = self._parse_kalshi_market(markets[0]) + if pm: + pm.probability_yes = best_yes + pm.probability_no = 1.0 - best_yes + return pm + + return None + except Exception as e: + logger.warning(f"Kalshi detail error for {ticker}: {e}") + return None + + # ── Limitless ─────────────────────────────────────────── + + async def _search_limitless(self, query: str) -> list[PredictionMarket]: + """Search Limitless Exchange markets by fetching active and filtering.""" + try: + resp = await self._http.get( + f"{LIMITLESS_BASE}/markets/active", + params={"limit": 25}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Limitless markets returned {resp.status_code}") + return [] + + data = resp.json() + all_markets = data.get("data", []) + + # Filter client-side by query terms + query_lower = query.lower() + query_terms = query_lower.split() + + results = [] + for m in all_markets: + title = m.get("title", "").lower() + if any(term in title for term in query_terms): + pm = self._parse_limitless_market(m) + if pm: + results.append(pm) + + return results + except Exception as e: + logger.warning(f"Limitless search error: {e}") + return [] + + def _parse_limitless_market(self, m: dict) -> PredictionMarket | None: + """Parse a Limitless market dict into unified PredictionMarket.""" + try: + title = m.get("title", "") + slug = m.get("slug", str(m.get("id", ""))) + + # Prices: [YES%, NO%] - e.g., [42.8, 57.2] + prices = m.get("prices", [50, 50]) + prob_yes = float(prices[0]) / 100 if isinstance(prices, list) and len(prices) >= 1 else 0.5 + prob_no = float(prices[1]) / 100 if isinstance(prices, list) and len(prices) >= 2 else 0.5 + + # Volume: use volumeFormatted if available, else volume + vol_str = m.get("volumeFormatted", str(m.get("volume", 0))) + volume = float(vol_str) if vol_str else 0.0 + + categories = m.get("categories", []) + category = categories[0] if categories else "" + tags = m.get("tags", []) + tokens_mentioned = _extract_token_symbols(title) + + return PredictionMarket( + source="limitless", + source_id=str(slug), + question=title, + slug=str(slug), + probability_yes=prob_yes, + probability_no=prob_no, + volume_usd=volume, + category=category, + tags=tags, + tokens_mentioned=tokens_mentioned, + url=f"https://limitless.exchange/markets/{slug}" if slug else "", + ends_at=m.get("expirationDate", ""), + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Limitless market: {e}") + return None + + async def _trending_limitless(self, limit: int) -> list[PredictionMarket]: + """Get trending Limitless markets.""" + try: + limit = min(limit, 25) # API max + resp = await self._http.get( + f"{LIMITLESS_BASE}/markets/active", + params={"limit": limit}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + if resp.status_code != 200: + return [] + + data = resp.json() + markets = data.get("data", []) + return [pm for m in markets[:limit] if (pm := self._parse_limitless_market(m))] + except Exception as e: + logger.warning(f"Limitless trending error: {e}") + return [] + + # ── Manifold ──────────────────────────────────────────── + + async def _search_manifold(self, query: str) -> list[PredictionMarket]: + """Search Manifold Markets (play-money, sentiment signals). + + Manifold is pure play-money but useful for: + - Forecasting community sentiment + - Early signal detection (top forecasters often move before real-money markets) + - Broad question coverage (more niche crypto questions than Polymarket) + """ + try: + resp = await self._http.get( + f"{MANIFOLD_BASE}/search-markets", + params={"term": query, "limit": 20}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Manifold search returned {resp.status_code}") + return [] + + data = resp.json() + contracts = data if isinstance(data, list) else data.get("contracts", data.get("markets", [])) + + results = [] + for c in contracts[:10]: + pm = self._parse_manifold_market(c) + if pm: + results.append(pm) + + return results + except Exception as e: + logger.warning(f"Manifold search error: {e}") + return [] + + def _parse_manifold_market(self, c: dict) -> PredictionMarket | None: + """Parse a Manifold contract into unified PredictionMarket.""" + try: + question = c.get("question", "") + slug = c.get("slug", c.get("id", "")) + prob = float(c.get("probability", c.get("prob", 0.5))) + volume = float(c.get("volume", c.get("volume24Hours", 0))) + + # Manifold uses "Mana" play money, volume is a signal but lower weight + tokens_mentioned = _extract_token_symbols(question) + tags = list(c.get("tags", [])) + + return PredictionMarket( + source="manifold", + source_id=str(slug), + question=question, + slug=str(slug), + probability_yes=prob, + probability_no=1.0 - prob, + volume_usd=volume, + category="", + tags=tags, + tokens_mentioned=tokens_mentioned, + url=f"https://manifold.markets/{c.get('creatorUsername', '')}/{slug}" if slug else "", + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Manifold market: {e}") + return None + + # ── Helpers ───────────────────────────────────────────── + + @staticmethod + def _parse_json_field(val): + """Parse double-encoded JSON fields (Polymarket Gamma API).""" + if isinstance(val, str): + try: + return json.loads(val) + except (json.JSONDecodeError, TypeError): + return val + return val + + +# ── Singleton ──────────────────────────────────────────────────── + +_service: PredictionMarketService | None = None + + +def get_prediction_market_service() -> PredictionMarketService: + """Get or create the singleton PredictionMarketService.""" + global _service + if _service is None: + _service = PredictionMarketService() + return _service + + +# ── Helpers: Token Extraction ──────────────────────────────────── + +# Common token symbols to detect in market questions +_COMMON_TOKENS = { + "BTC", + "ETH", + "SOL", + "USDT", + "USDC", + "DAI", + "BNB", + "XRP", + "ADA", + "DOGE", + "MATIC", + "POL", + "DOT", + "AVAX", + "LINK", + "UNI", + "AAVE", + "ARB", + "OP", + "SUI", + "APT", + "TIA", + "SEI", + "STRK", + "WLD", + "PEPE", + "SHIB", + "BONK", + "WIF", + "JUP", + "PYTH", + "RNDR", + "FET", + "AGIX", + "OCEAN", + "IMX", + "INJ", + "EIGEN", + "ENA", + "ETHFI", +} + +# Broader project names often referenced in prediction markets +_COMMON_PROJECTS = { + "polymarket", + "kalshi", + "manifold", + "uniswap", + "sushiswap", + "aave", + "compound", + "makerdao", + "maker", + "lido", + "eigenlayer", + "chainlink", + "arbitrum", + "optimism", + "polygon", + "avalanche", + "fantom", + "near", + "celestia", + "worldcoin", + "tether", + "circle", + "coinbase", + "binance", + "kraken", + "ftx", + "celcius", + "blockfi", + "three arrows", + "alameda", + "jump crypto", + "wintermute", + "curve", + "balancer", + "thorchain", + "osmosis", + "dydx", + "gmx", + "hyperliquid", + "jupiter", + "raydium", + "orca", + "wormhole", + "layerzero", + "zksync", + "starknet", + "scroll", + "linea", + "base", + "mantle", + "mode", + "blast", +} + + +def _extract_token_symbols(text: str) -> list[str]: + """Extract known token symbols and project names from text.""" + found = [] + text_upper = text.upper() + text_lower = text.lower() + + # Check token symbols (typically uppercase in text) + for token in _COMMON_TOKENS: + # Match as word boundary: " BTC " or "BTC's" or "$BTC" + if ( + f" {token} " in f" {text_upper} " + or f"${token}" in text_upper + or text_upper.startswith(f"{token} ") + or text_upper.endswith(f" {token}") + ) and token not in found: + found.append(token) + + # Check project names (case-insensitive) + for project in _COMMON_PROJECTS: + if project in text_lower and project.upper() not in found: + found.append(project) + + return found + + +# ── Caching Helpers ────────────────────────────────────────────── + + +def _cache_hash(*args) -> str: + """Create a short hash for cache keys.""" + raw = "|".join(str(a) for a in args) + return hashlib.md5(raw.encode()).hexdigest()[:12] + + +def _market_to_dict(m: PredictionMarket) -> dict: + """Serialize PredictionMarket to dict for JSON caching.""" + return { + "source": m.source, + "source_id": m.source_id, + "question": m.question, + "slug": m.slug, + "probability_yes": m.probability_yes, + "probability_no": m.probability_no, + "volume_usd": m.volume_usd, + "liquidity_usd": m.liquidity_usd, + "category": m.category, + "tags": m.tags, + "tokens_mentioned": m.tokens_mentioned, + "is_security_relevant": m.is_security_relevant, + "is_crypto_relevant": m.is_crypto_relevant, + "url": m.url, + "ends_at": m.ends_at, + "updated_at": m.updated_at, + } + + +def _dict_to_market(d: dict) -> PredictionMarket: + """Deserialize dict back to PredictionMarket.""" + return PredictionMarket( + source=d.get("source", ""), + source_id=d.get("source_id", ""), + question=d.get("question", ""), + slug=d.get("slug", ""), + probability_yes=float(d.get("probability_yes", 0.5)), + probability_no=float(d.get("probability_no", 0.5)), + volume_usd=float(d.get("volume_usd", 0)), + liquidity_usd=float(d.get("liquidity_usd", 0)), + category=d.get("category", ""), + tags=d.get("tags", []), + tokens_mentioned=d.get("tokens_mentioned", []), + is_security_relevant=d.get("is_security_relevant", False), + is_crypto_relevant=d.get("is_crypto_relevant", False), + url=d.get("url", ""), + ends_at=d.get("ends_at", ""), + updated_at=d.get("updated_at", ""), + ) + + +def _digest_to_dict(d: PredictionDigest) -> dict: + """Serialize PredictionDigest to dict for JSON caching.""" + return { + "generated_at": d.generated_at, + "total_markets_searched": d.total_markets_searched, + "security_relevant_count": d.security_relevant_count, + "crypto_relevant_count": d.crypto_relevant_count, + "top_threats": [_market_to_dict(m) for m in d.top_threats], + "token_specific_markets": [_market_to_dict(m) for m in d.token_specific_markets], + "ecosystem_risk_markets": [_market_to_dict(m) for m in d.ecosystem_risk_markets], + "regulatory_markets": [_market_to_dict(m) for m in d.regulatory_markets], + } + + +def _dict_to_digest(d: dict) -> PredictionDigest: + """Deserialize dict back to PredictionDigest.""" + return PredictionDigest( + generated_at=d.get("generated_at", ""), + total_markets_searched=d.get("total_markets_searched", 0), + security_relevant_count=d.get("security_relevant_count", 0), + crypto_relevant_count=d.get("crypto_relevant_count", 0), + top_threats=[_dict_to_market(m) for m in d.get("top_threats", [])], + token_specific_markets=[_dict_to_market(m) for m in d.get("token_specific_markets", [])], + ecosystem_risk_markets=[_dict_to_market(m) for m in d.get("ecosystem_risk_markets", [])], + regulatory_markets=[_dict_to_market(m) for m in d.get("regulatory_markets", [])], + ) diff --git a/app/domains/mcp/__init__.py b/app/domains/mcp/__init__.py new file mode 100644 index 0000000..458132e --- /dev/null +++ b/app/domains/mcp/__init__.py @@ -0,0 +1,33 @@ +"""MCP domain - public API. + +Phase 4 domain consolidation: app.mcp -> app.domains.mcp. +""" +from __future__ import annotations + +from app.domains.mcp.registry import TOOL_CATEGORIES, get_tool_by_name +from app.domains.mcp.server import ( + MCP_PROTOCOL_VERSION, + MCP_SERVER_VERSION, + TOOL_CATALOG, + TOOL_DEPRECATED, + TOOL_SUCCESSORS, + TOOL_VERSIONS, + call_tool, +) +from app.domains.mcp.x402_mcp_server import TOOLS, handle_mcp_call +from app.domains.mcp.x402_tool_manager import X402ToolManager + +__all__ = [ + "MCP_PROTOCOL_VERSION", + "MCP_SERVER_VERSION", + "TOOLS", + "TOOL_CATALOG", + "TOOL_CATEGORIES", + "TOOL_DEPRECATED", + "TOOL_SUCCESSORS", + "TOOL_VERSIONS", + "X402ToolManager", + "call_tool", + "get_tool_by_name", + "handle_mcp_call", +] diff --git a/app/mcp/manifest.py b/app/domains/mcp/manifest.py similarity index 100% rename from app/mcp/manifest.py rename to app/domains/mcp/manifest.py diff --git a/app/mcp/registry.py b/app/domains/mcp/registry.py similarity index 98% rename from app/mcp/registry.py rename to app/domains/mcp/registry.py index 8f60fc3..0b5330b 100644 --- a/app/mcp/registry.py +++ b/app/domains/mcp/registry.py @@ -4,8 +4,8 @@ from __future__ import annotations import logging -from app.mcp.manifest import MCPServerManifest, MCPToolManifest -from app.mcp.server import ( +from app.domains.mcp.manifest import MCPServerManifest, MCPToolManifest +from app.domains.mcp.server import ( TOOL_CATALOG, TOOL_DEPRECATED, TOOL_SUCCESSORS, diff --git a/app/api/v1/mcp/router.py b/app/domains/mcp/router.py similarity index 99% rename from app/api/v1/mcp/router.py rename to app/domains/mcp/router.py index 9e95279..c230990 100644 --- a/app/api/v1/mcp/router.py +++ b/app/domains/mcp/router.py @@ -18,7 +18,7 @@ from typing import Any from fastapi import APIRouter, Request from pydantic import BaseModel -from app.mcp.server import ( +from app.domains.mcp.server import ( MCP_PROTOCOL_VERSION, MCP_SERVER_VERSION, TOOL_CATALOG, diff --git a/app/mcp/server.py b/app/domains/mcp/server.py similarity index 99% rename from app/mcp/server.py rename to app/domains/mcp/server.py index 3c165c7..5199653 100644 --- a/app/mcp/server.py +++ b/app/domains/mcp/server.py @@ -601,7 +601,7 @@ async def call_tool(name: str, arguments: dict) -> dict: limit = min(int(arguments.get("limit", 1000)), 10000) try: - from app.mcp.tools.eth_labels_tool import query_eth_labels_db_mcp + from app.domains.mcp.tools.eth_labels_tool import query_eth_labels_db_mcp result = await query_eth_labels_db_mcp(sql, limit) return {"result": result} @@ -611,7 +611,7 @@ async def call_tool(name: str, arguments: dict) -> dict: if name == "eth_labels_stats": # T22: Get statistics about eth-labels.db try: - from app.mcp.tools.eth_labels_tool import get_eth_labels_stats_mcp + from app.domains.mcp.tools.eth_labels_tool import get_eth_labels_stats_mcp result = await get_eth_labels_stats_mcp() return {"result": result} @@ -620,7 +620,7 @@ async def call_tool(name: str, arguments: dict) -> dict: if name == "mcp_discover": # MCP catalog discovery with versioning + deprecation + category filtering - from app.mcp.registry import TOOL_CATEGORIES # lazy: avoid circular import + from app.domains.mcp.registry import TOOL_CATEGORIES # lazy: avoid circular import include_deprecated = arguments.get("include_deprecated", False) category = arguments.get("category") diff --git a/app/mcp/tools/eth_labels_tool.py b/app/domains/mcp/tools/eth_labels_tool.py similarity index 100% rename from app/mcp/tools/eth_labels_tool.py rename to app/domains/mcp/tools/eth_labels_tool.py diff --git a/app/mcp/x402_ai_guard.py b/app/domains/mcp/x402_ai_guard.py similarity index 100% rename from app/mcp/x402_ai_guard.py rename to app/domains/mcp/x402_ai_guard.py diff --git a/app/mcp/x402_bundles.py b/app/domains/mcp/x402_bundles.py similarity index 100% rename from app/mcp/x402_bundles.py rename to app/domains/mcp/x402_bundles.py diff --git a/app/mcp/x402_mcp_server.py b/app/domains/mcp/x402_mcp_server.py similarity index 100% rename from app/mcp/x402_mcp_server.py rename to app/domains/mcp/x402_mcp_server.py diff --git a/app/mcp/x402_tool_manager.py b/app/domains/mcp/x402_tool_manager.py similarity index 100% rename from app/mcp/x402_tool_manager.py rename to app/domains/mcp/x402_tool_manager.py diff --git a/app/domains/referral/__init__.py b/app/domains/referral/__init__.py new file mode 100644 index 0000000..5bd3cda --- /dev/null +++ b/app/domains/referral/__init__.py @@ -0,0 +1,31 @@ +"""Referral domain - public API. + +Phase 4 domain consolidation: extracted from telegram/rugmunchbot/bot.py. +""" +from __future__ import annotations + +from app.domains.referral.core import ( + MILESTONES, + REFERRAL_BONUS_SCANS, + apply_referral_code, + generate_referral_link, + get_referral_stats, +) +from app.domains.referral.partners import ( + DEX_REF_LINKS, + PARTNER_CODES, + dex_buttons, + format_dex_url, +) + +__all__ = [ + "DEX_REF_LINKS", + "MILESTONES", + "PARTNER_CODES", + "REFERRAL_BONUS_SCANS", + "apply_referral_code", + "dex_buttons", + "format_dex_url", + "generate_referral_link", + "get_referral_stats", +] diff --git a/app/domains/referral/core.py b/app/domains/referral/core.py new file mode 100644 index 0000000..9f115d5 --- /dev/null +++ b/app/domains/referral/core.py @@ -0,0 +1,77 @@ +"""Referral domain - Telegram bot referral system. + +Phase 4 domain consolidation: extracted from telegram/rugmunchbot/bot.py. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import sqlite3 + +REFERRAL_BONUS_SCANS = 5 + +MILESTONES = {5: "🥉", 10: "🥈", 25: "🥇", 50: "💎", 100: "👑"} + + +def apply_referral_code( + user_id: int, + ref_code: str, + user_referral_code: str | None, + referred_by: str | None, + conn: sqlite3.Connection, +) -> bool: + """Apply a referral code for a new/existing user. + + Returns True if the referral was successfully applied. + """ + if not ref_code or ref_code == user_referral_code or referred_by: + return False + + referrer = conn.execute( + "SELECT user_id FROM users WHERE referral_code = ?", (ref_code,) + ).fetchone() + if not referrer: + return False + + conn.execute( + "UPDATE users SET referred_by = ?, bonus_scans = bonus_scans + ? WHERE user_id = ?", + (ref_code, REFERRAL_BONUS_SCANS, user_id), + ) + conn.execute( + "UPDATE users SET bonus_scans = bonus_scans + ? WHERE user_id = ?", + (REFERRAL_BONUS_SCANS, referrer["user_id"]), + ) + conn.commit() + return True + + +def get_referral_stats(user_id: int, referral_code: str, conn: sqlite3.Connection) -> dict[str, Any]: + """Return referral stats for a user. + + Includes total referrals, bonus scans earned, next milestone info, and share link. + """ + count_row = conn.execute( + "SELECT COUNT(*) as c FROM users WHERE referred_by = ?", (referral_code,) + ).fetchone() + count = count_row["c"] if count_row else 0 + + next_ms = next((k for k in sorted(MILESTONES.keys()) if k > count), None) + ms_text = ( + f"\n🎯 Next milestone: {MILESTONES[next_ms]} {next_ms} referrals" + if next_ms + else "\n👑 You've hit all milestones!" + ) + + return { + "user_id": user_id, + "referral_code": referral_code, + "count": count, + "bonus_earned": count * REFERRAL_BONUS_SCANS, + "next_milestone_text": ms_text, + } + + +def generate_referral_link(bot_username: str, referral_code: str) -> str: + """Generate a Telegram referral start link.""" + return f"https://t.me/{bot_username}?start=ref_{referral_code}" diff --git a/app/domains/referral/partners.py b/app/domains/referral/partners.py new file mode 100644 index 0000000..81349b2 --- /dev/null +++ b/app/domains/referral/partners.py @@ -0,0 +1,93 @@ +"""Referral partner links and helpers. + +Phase 4 domain consolidation: DEX/DeFi partner referral data extracted from +telegram bot config so it can be reused by scanners, web, and Telegram. +""" +from __future__ import annotations + +from telegram import InlineKeyboardButton + +# Partner referral codes / handles. Keep canonical here. +PARTNER_CODES = { + "jupiter": "rugmunch", + "photon": "rugmunch", + "bullx": "rugmunch", + "gmgn": "rugmunch", + "birdeye": "rugmunch", + "oneinch": "rugmunch", + "pancakeswap": "rugmunch", +} + +DEX_REF_LINKS = { + "jupiter": { + "name": "Jupiter", + "emoji": "🪐", + "url": "https://jup.ag/swap/SOL-{token}?ref=rugmunch", + "chains": ["solana"], + "description": "Best Solana DEX aggregator", + }, + "photon": { + "name": "Photon", + "emoji": "⚡", + "url": "https://photon-sol.tinyastro.io/en/lp/{token}?handle=rugmunch", + "chains": ["solana", "ethereum", "base"], + "description": "Fast Solana trading terminal", + }, + "bullx": { + "name": "BullX", + "emoji": "🐂", + "url": "https://neo.bullx.io/terminal?chain={chain}&address={token}&ref=rugmunch", + "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], + "description": "Multi-chain trading bot", + }, + "gmgn": { + "name": "GMGN.ai", + "emoji": "📊", + "url": "https://gmgn.ai/sol/token/{token}?ref=rugmunch", + "chains": ["solana", "ethereum", "base"], + "description": "Smart money tracking + trading", + }, + "birdeye": { + "name": "Birdeye", + "emoji": "🦅", + "url": "https://birdeye.so/token/{token}?ref=rugmunch", + "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], + "description": "Token analytics & charts", + }, + "oneinch": { + "name": "1inch", + "emoji": "🦄", + "url": "https://app.1inch.io/#/{chain_id}/simple/swap/ETH/{token}?ref=rugmunch", + "chains": ["ethereum", "base", "arbitrum", "bsc", "polygon", "avalanche"], + "description": "Best EVM DEX aggregator", + }, + "pancakeswap": { + "name": "PancakeSwap", + "emoji": "🥞", + "url": "https://pancakeswap.finance/swap?inputCurrency=BNB&outputCurrency={token}&ref=rugmunch", + "chains": ["bsc", "ethereum", "arbitrum", "base"], + "description": "BSC's #1 DEX", + }, +} + + +def format_dex_url(platform: str, token: str, chain: str, chain_id: int) -> str | None: + """Return a formatted referral URL for a DEX partner, or None if unsupported.""" + dex = DEX_REF_LINKS.get(platform) + if not dex or chain.lower() not in dex.get("chains", []): + return None + return dex["url"].format(token=token, chain=chain, chain_id=chain_id) + + +def dex_buttons(token: str, chain: str, chain_id: int) -> list[InlineKeyboardButton]: + """Generate Telegram inline buttons for DEX partners supporting the given chain.""" + buttons = [] + for _key, dex in DEX_REF_LINKS.items(): + if chain.lower() in dex.get("chains", []): + url = dex["url"].format( + token=token, + chain=chain, + chain_id=chain_id, + ) + buttons.append(InlineKeyboardButton(f"{dex['emoji']} {dex['name']}", url=url)) + return buttons diff --git a/app/domains/telegram/rugmunchbot/bot.py b/app/domains/telegram/rugmunchbot/bot.py index fc4cc3c..41c26a2 100644 --- a/app/domains/telegram/rugmunchbot/bot.py +++ b/app/domains/telegram/rugmunchbot/bot.py @@ -42,6 +42,12 @@ from telegram.ext import ( ) # ── Local imports ── +from app.domains.referral import ( + apply_referral_code, + dex_buttons, + generate_referral_link, + get_referral_stats, +) from app.domains.telegram.rugmunchbot import db from app.domains.telegram.rugmunchbot.config import ( BACKEND_URL, @@ -51,7 +57,6 @@ from app.domains.telegram.rugmunchbot.config import ( BOT_USERNAME, CHAIN_IDS, CHANNELS, - DEX_REF_LINKS, DEXSCREENER, GOPLUS, HONEYPOT, @@ -157,18 +162,6 @@ def web_scan_button(address: str, chain: str = "") -> InlineKeyboardButton: return InlineKeyboardButton("🌐 Full Report on RugMunch.io", url=url) -def dex_buttons(token: str, chain: str) -> list[InlineKeyboardButton]: - buttons = [] - for _key, dex in DEX_REF_LINKS.items(): - if chain.lower() in dex.get("chains", []): - url = dex["url"].format( - token=token, - chain=chain, - chain_id=CHAIN_IDS.get(chain.lower(), 1), - ) - buttons.append(InlineKeyboardButton(f"{dex['emoji']} {dex['name']}", url=url)) - return buttons - def social_proof_text() -> str: """Dynamic social proof from DB stats.""" @@ -670,7 +663,7 @@ def scan_result_kb(address: str, chain: str) -> InlineKeyboardMarkup: InlineKeyboardButton("⭐ Watch", callback_data=f"watch_{address}_{chain}"), ], ] - dex_btns = dex_buttons(address, chain) + dex_btns = dex_buttons(address, chain, CHAIN_IDS.get(chain.lower(), 1)) for i in range(0, len(dex_btns), 2): rows.append(dex_btns[i : i + 2]) rows.append([InlineKeyboardButton("◀️ Menu", callback_data="menu_main")]) @@ -747,19 +740,16 @@ async def cmd_start(update: Update, ctx: ContextTypes.DEFAULT_TYPE): # Referral handling if ctx.args and ctx.args[0].startswith("ref_"): ref_code = ctx.args[0][4:] - if ref_code != u.get("referral_code") and not u.get("referred_by"): - conn = db.get_db() - referrer = conn.execute("SELECT user_id FROM users WHERE referral_code = ?", (ref_code,)).fetchone() - if referrer: - conn.execute( - "UPDATE users SET referred_by = ?, bonus_scans = bonus_scans + 5 WHERE user_id = ?", - (ref_code, user.id), - ) - conn.execute( - "UPDATE users SET bonus_scans = bonus_scans + 5 WHERE user_id = ?", - (referrer["user_id"],), - ) - conn.commit() + conn = db.get_db() + try: + apply_referral_code( + user.id, + ref_code, + u.get("referral_code"), + u.get("referred_by"), + conn, + ) + finally: conn.close() proof = social_proof_text() @@ -1346,24 +1336,19 @@ async def cmd_refer(update: Update, ctx: ContextTypes.DEFAULT_TYPE): user = update.effective_user u = db.get_or_create_user(user.id, user.username, user.first_name) ref = u.get("referral_code", "UNKNOWN") - link = f"https://t.me/{BOT_USERNAME}?start=ref_{ref}" + link = generate_referral_link(BOT_USERNAME, ref) conn = db.get_db() - count = conn.execute("SELECT COUNT(*) as c FROM users WHERE referred_by = ?", (ref,)).fetchone()["c"] - conn.close() - milestones = {5: "🥉", 10: "🥈", 25: "🥇", 50: "💎", 100: "👑"} - next_ms = next((k for k in sorted(milestones.keys()) if k > count), None) - ms_text = ( - f"\n🎯 Next milestone: {milestones[next_ms]} {next_ms} referrals" - if next_ms - else "\n👑 You've hit all milestones!" - ) + try: + stats = get_referral_stats(user.id, ref, conn) + finally: + conn.close() text = ( f"🤝 Refer & Earn\n{sep()}\n\n" f"Invite friends and both get 5 bonus scans!\n\n" f"🔗 Your Link:\n{link}\n\n" f"📊 Your Stats:\n" - f" Friends Referred: {count}\n" - f" Bonus Earned: {count * 5} scans{ms_text}\n\n" + f" Friends Referred: {stats['count']}\n" + f" Bonus Earned: {stats['bonus_earned']} scans{stats['next_milestone_text']}\n\n" f"How it works:\n" f" 1. Share your referral link\n" f" 2. Friend starts the bot via your link\n" @@ -1865,7 +1850,7 @@ async def handle_callback(update: Update, ctx: ContextTypes.DEFAULT_TYPE): await query.edit_message_text(text, parse_mode=ParseMode.HTML, reply_markup=back_kb()) elif data == "menu_refer": u = db.get_or_create_user(user.id) - link = f"https://t.me/{BOT_USERNAME}?start=ref_{u.get('referral_code', '')}" + link = generate_referral_link(BOT_USERNAME, u.get("referral_code", "")) await query.edit_message_text( f"🤝 Refer Friends\n\nShare your link:\n{link}\n\nBoth get 5 bonus scans!", parse_mode=ParseMode.HTML, diff --git a/app/domains/telegram/rugmunchbot/config.py b/app/domains/telegram/rugmunchbot/config.py index 443da5a..7360aeb 100644 --- a/app/domains/telegram/rugmunchbot/config.py +++ b/app/domains/telegram/rugmunchbot/config.py @@ -7,6 +7,8 @@ Tiers, channels, API endpoints, DEX ref links, and constants. import os +from app.domains.referral.partners import DEX_REF_LINKS # noqa: F401 + # ══════════════════════════════════════════════ # BOT # ══════════════════════════════════════════════ @@ -263,58 +265,7 @@ SCAN_PACKS = { # DEX REFERRAL LINKS # ══════════════════════════════════════════════ # These generate revenue when users trade through our links. -# Format: {platform: {url_template, chains, emoji, description}} -DEX_REF_LINKS = { - "jupiter": { - "name": "Jupiter", - "emoji": "🪐", - "url": "https://jup.ag/swap/SOL-{token}?ref=rugmunch", - "chains": ["solana"], - "description": "Best Solana DEX aggregator", - }, - "photon": { - "name": "Photon", - "emoji": "⚡", - "url": "https://photon-sol.tinyastro.io/en/lp/{token}?handle=rugmunch", - "chains": ["solana", "ethereum", "base"], - "description": "Fast Solana trading terminal", - }, - "bullx": { - "name": "BullX", - "emoji": "🐂", - "url": "https://neo.bullx.io/terminal?chain={chain}&address={token}&ref=rugmunch", - "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], - "description": "Multi-chain trading bot", - }, - "gmgn": { - "name": "GMGN.ai", - "emoji": "📊", - "url": "https://gmgn.ai/sol/token/{token}?ref=rugmunch", - "chains": ["solana", "ethereum", "base"], - "description": "Smart money tracking + trading", - }, - "birdeye": { - "name": "Birdeye", - "emoji": "🦅", - "url": "https://birdeye.so/token/{token}?ref=rugmunch", - "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], - "description": "Token analytics & charts", - }, - "oneinch": { - "name": "1inch", - "emoji": "🦄", - "url": "https://app.1inch.io/#/{chain_id}/simple/swap/ETH/{token}?ref=rugmunch", - "chains": ["ethereum", "base", "arbitrum", "bsc", "polygon", "avalanche"], - "description": "Best EVM DEX aggregator", - }, - "pancakeswap": { - "name": "PancakeSwap", - "emoji": "🥞", - "url": "https://pancakeswap.finance/swap?inputCurrency=BNB&outputCurrency={token}&ref=rugmunch", - "chains": ["bsc", "ethereum", "arbitrum", "base"], - "description": "BSC's #1 DEX", - }, -} +# Canonical definitions live in app.domains.referral.partners. # Chain ID mapping for 1inch CHAIN_IDS = { diff --git a/app/intel_pipeline.py b/app/intel_pipeline.py index ff2c30f..58b30bd 100644 --- a/app/intel_pipeline.py +++ b/app/intel_pipeline.py @@ -1,298 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.intel_pipeline. + +Migrate imports from `app.intel_pipeline` to `app.domains.intelligence.intel_pipeline`. +This shim will be removed in Phase 3 cleanup. """ -RMI Intelligence Pipeline - HF + Supabase + RAG -================================================ -Unified intelligence service: scam classification (HF models), -wallet labeling via RAG pattern matching, Supabase hybrid storage. -""" +from __future__ import annotations -import hashlib -import logging -import os -from datetime import UTC, datetime - -import httpx -from dotenv import load_dotenv - -load_dotenv("/app/.env", override=True) - -logger = logging.getLogger(__name__) - -HF_TOKEN = os.getenv("HF_TOKEN", "") -HF_API = "https://api-inference.huggingface.co/models" - - -def _get_url(): - return os.getenv("SUPABASE_URL", "") - - -def _get_key(): - return os.getenv("SUPABASE_SERVICE_KEY", "") - - -def _get_headers(): - key = _get_key() - return { - "apikey": key, - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - -# ── HF Models (Paywalled - using local fallback) ────── -# HF Inference API now requires PRO subscription ($9/mo). -# Using local pattern matching + RAG memory instead. -# Enable HF by setting HUGGINGFACE_TOKEN to a valid PRO key. - -SCAM_LABELS = ["scam", "rugpull", "honeypot", "legitimate", "phishing", "ponzi"] - -SCAM_KEYWORDS = { - "scam": ["scam", "fraud", "stole", "stolen", "exit scam", "fake"], - "rugpull": ["rug", "rugpull", "pulled liquidity", "drained", "removed liquidity", "lp removed"], - "honeypot": [ - "honeypot", - "cannot sell", - "can't sell", - "unable to sell", - "transfer disabled", - "sell tax 100", - ], - "phishing": [ - "phishing", - "airdrop scam", - "claim reward", - "verify wallet", - "seed phrase", - "private key", - ], - "ponzi": [ - "ponzi", - "mlm", - "multi level", - "referral rewards", - "guaranteed returns", - "double your", - ], - "insider": ["insider", "team wallet", "dev wallet", "pre-sale", "unlocked tokens", "vesting"], -} - - -async def classify_scam_risk(text: str) -> dict: - """Classify scam risk using local pattern matching (HF paywalled).""" - text_lower = text.lower() - scores = {} - - for label, keywords in SCAM_KEYWORDS.items(): - score = sum(1 for kw in keywords if kw in text_lower) - if score > 0: - scores[label] = min(score / len(keywords), 1.0) - - try: - from app.rag_service import search_documents - - rag_results = await search_documents("known_scams", text, limit=3) - for r in rag_results: - content = r.get("content", "").lower() - for label, keywords in SCAM_KEYWORDS.items(): - if any(kw in content for kw in keywords): - scores[label] = scores.get(label, 0) + 0.1 - except Exception: - pass - - if not scores: - return { - "risk": "low", - "labels": {}, - "confidence": 0, - "is_scam": False, - "risk_level": "low", - "model": "local_pattern_match", - } - - top = max(scores, key=scores.get) - top_score = scores[top] - is_scam = top in ("scam", "rugpull", "honeypot", "phishing", "ponzi") - - return { - "model": "local_pattern_match", - "labels": {k: round(v, 3) for k, v in sorted(scores.items(), key=lambda x: x[1], reverse=True)}, - "top_label": top, - "confidence": round(top_score, 3), - "is_scam": is_scam, - "risk_level": "high" if (is_scam and top_score > 0.5) else "medium" if is_scam else "low", - } - - -async def generate_embedding(text: str) -> list[float] | None: - """Generate embedding vector for RAG storage using HF free tier.""" - if not HF_TOKEN: - return None - - async with httpx.AsyncClient(timeout=60) as client: - r = await client.post( - f"{HF_API}/{EMBEDDING_MODEL}", # noqa: F821 -- pre-existing bug, see fix(f821) tracking issue - headers={"Authorization": f"Bearer {HF_TOKEN}"}, - json={"inputs": text[:1024]}, - ) - if r.status_code == 503: - logger.warning("HF embedding cold start, model loading") - return None - if r.status_code == 200: - result = r.json() - # Handle different response formats - if isinstance(result, list) and len(result) > 0: - return result[0] if isinstance(result[0], list) else result - return result - logger.warning(f"HF embedding failed: {r.status_code}") - return None - - -# ── Wallet Labeling via Pattern Memory ────────────────── - -WALLET_PATTERNS = { - "sybil_farmer": [ - "funded from exchange within seconds of 100+ other wallets", - "identical funding amounts across multiple wallets", - "no organic activity, only test transactions", - "funded by known Sybil distributor", - ], - "wash_trader": [ - "circular transfers between related wallets", - "buys and sells same token within minutes", - "volume spikes without holder count changes", - "coordinated buy/sell patterns", - ], - "sandwich_bot": [ - "high frequency trading with frontrun pattern", - "buys before large buys, sells immediately after", - "MEV extraction patterns", - "flashbots bundle usage", - ], - "liquidity_remover": [ - "large LP removal shortly after token launch", - "multiple LP positions removed simultaneously", - "liquidity drained to fresh wallet", - "LP removal preceded by marketing push", - ], - "honeypot_deployer": [ - "deploys tokens that can't be sold", - "reuses contract code across multiple tokens", - "disables transfers after liquidity added", - "ownership not renounced, hidden mint functions", - ], - "phishing_operator": [ - "sends tokens to many addresses with scam links", - "impersonates legitimate token contracts", - "uses airdrop as phishing vector", - "connects to known phishing domains", - ], - "mixer_user": [ - "funds pass through Tornado Cash or similar", - "receives from mixer, sends to clean wallet", - "layered mixing through multiple hops", - "funds originate from high-risk sources", - ], - "insider_trader": [ - "buys tokens before public announcements", - "linked to team wallets or deployers", - "sells immediately after hype peak", - "coordinated timing with other insiders", - ], -} - - -async def label_wallet(wallet_data: dict) -> dict: - """Label a wallet based on behavioral patterns and RAG memory.""" - labels = [] - confidence_scores = {} - - # Check behavioral patterns - behavior = wallet_data.get("behavior_summary", "") - wallet_data.get("transactions", []) - - for label, patterns in WALLET_PATTERNS.items(): - score = 0 - for pattern in patterns: - if pattern.lower() in behavior.lower(): - score += 1 - if score > 0: - confidence = min(score / len(patterns), 1.0) - confidence_scores[label] = round(confidence, 2) - if confidence > 0.3: - labels.append({"label": label, "confidence": round(confidence, 2)}) - - # Query RAG for similar wallet patterns - try: - from app.rag_service import search_documents - - rag_results = await search_documents( - "wallet_profiles", wallet_data.get("address", "") + " " + behavior, limit=5 - ) - if rag_results: - for r in rag_results: - content = r.get("content", "") - for label, patterns in WALLET_PATTERNS.items(): - if any(p.lower() in content.lower() for p in patterns): - if label not in confidence_scores: - confidence_scores[label] = 0 - confidence_scores[label] = min(confidence_scores[label] + 0.15, 1.0) - except Exception: - pass - - labels = [ - {"label": k, "confidence": v} - for k, v in sorted(confidence_scores.items(), key=lambda x: x[1], reverse=True) - if v > 0.2 - ] - - return { - "wallet_address": wallet_data.get("address", "unknown"), - "labels": labels[:5], - "primary_label": labels[0]["label"] if labels else "unknown", - "risk_score": max([line["confidence"] for line in labels]) * 100 if labels else 0, - "analyzed_at": datetime.now(UTC).isoformat(), - } - - -# ── Supabase Hybrid Storage ──────────────────────────── - - -async def sync_to_supabase(collection: str, document: dict) -> dict: - """Sync RAG document to Supabase for persistent hybrid storage.""" - if not _get_url() or not _get_key(): - return {"status": "skipped", "reason": "No Supabase config"} - - doc_id = hashlib.sha256(f"{collection}:{document.get('content', '')[:100]}".encode()).hexdigest()[:16] - - payload = { - "document_id": doc_id, - "collection": collection, - "content": document.get("content", "")[:5000], - "metadata": document.get("metadata", {}), - "synced_at": datetime.now(UTC).isoformat(), - } - - headers = _get_headers() - headers["Prefer"] = "resolution=merge-duplicates" - - async with httpx.AsyncClient(timeout=15) as client: - r = await client.post(f"{_get_url()}/rest/v1/rag_documents", json=payload, headers=headers) - if r.status_code in (200, 201, 409): - return {"status": "synced", "doc_id": doc_id, "supabase_status": r.status_code} - return {"status": "failed", "error": r.text[:200]} - - -# ── Batch Processing ─────────────────────────────────── - - -async def run_intelligence_cycle() -> dict: - """Run a full intelligence cycle: classify, label, embed, sync.""" - results = { - "cycle": datetime.now(UTC).isoformat(), - "scam_checks": 0, - "wallet_labels": 0, - "embeddings": 0, - "supabase_syncs": 0, - "errors": [], - } - - return results +from app.domains.intelligence.intel_pipeline import * # noqa: F403 diff --git a/app/mcp/__init__.py b/app/mcp/__init__.py index ac13c71..593e3b3 100644 --- a/app/mcp/__init__.py +++ b/app/mcp/__init__.py @@ -1,10 +1,36 @@ -# MCP Server Modules -# ================== -# x402 gateways run on Cloudflare Workers (7 chains, 45 tools each) -# The port 8001 local server was shut down - CF Workers handle everything -# Backend is at /srv/rugmuncher-backend/backend/ (Docker container rmi_backend) +"""Deprecated shim - re-exports app.domains.mcp public API. -# Legacy reference only - do not import for production use -# from .x402_mcp_server import X402MCPServer, X402ToolManager +Migrate imports from `app.mcp.*` to `app.domains.mcp.*`. +This shim will be removed in Phase 3 cleanup. +""" +from __future__ import annotations -__all__ = [] +from app.domains.mcp import ( + MCP_PROTOCOL_VERSION, + MCP_SERVER_VERSION, + TOOL_CATALOG, + TOOL_CATEGORIES, + TOOL_DEPRECATED, + TOOL_SUCCESSORS, + TOOL_VERSIONS, + TOOLS, + X402ToolManager, + call_tool, + get_tool_by_name, + handle_mcp_call, +) + +__all__ = [ + "MCP_PROTOCOL_VERSION", + "MCP_SERVER_VERSION", + "TOOLS", + "TOOL_CATALOG", + "TOOL_CATEGORIES", + "TOOL_DEPRECATED", + "TOOL_SUCCESSORS", + "TOOL_VERSIONS", + "X402ToolManager", + "call_tool", + "get_tool_by_name", + "handle_mcp_call", +] diff --git a/app/meme_intelligence.py b/app/meme_intelligence.py index 7201df4..e83487c 100644 --- a/app/meme_intelligence.py +++ b/app/meme_intelligence.py @@ -1,423 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.meme_intelligence. + +Migrate imports from `app.meme_intelligence` to `app.domains.intelligence.meme_intelligence`. +This shim will be removed in Phase 3 cleanup. """ -Meme Intelligence Platform - Meme Coin Tracking, Smart Money in Memes, -Big Wins/Losses, KOL Scorecards, Social Monitoring. +from __future__ import annotations -Integrations: -- DexScreener: meme token launches, trending -- LunarCrush: social sentiment, social dominance -- X/Twitter: KOL posts, viral content -- Telegram: channel monitoring, group sentiment -- Arkham: whale tracking in memes -- Birdeye: Solana meme tokens -- Pump.fun: new meme launches -""" - -import logging -import os - -from dotenv import load_dotenv - -load_dotenv("/app/.env", override=True) -from datetime import UTC, datetime, timedelta # noqa: E402 - -logger = logging.getLogger(__name__) - -# ── Meme Intelligence Data Structures ───────────────────────── - -MEME_CHAINS = ["solana", "ethereum", "base", "bsc", "arbitrum"] - -MEME_CATEGORIES = { - "dog": ["dog", "doge", "shib", "akita", "kishu"], - "cat": ["cat", "pepe", "mog", "meow"], - "politi": ["trump", "biden", "maga", "politics"], - "celeb": ["celeb", "influencer", "famous"], - "ai": ["ai", "gpt", "neural", "chat"], - "gaming": ["game", "gaming", "nft", "metaverse"], - "other": [], -} - -# KOL Database Schema -KOL_DATABASE = { - # Example structure - populate from research - "twitter_handles": [], - "telegram_channels": [], - "wallet_addresses": [], - "track_record": {}, # past calls, win rate - "follower_counts": {}, - "engagement_rates": {}, -} - -# ── Meme Token Intelligence ──────────────────────────────────── - - -async def get_meme_trending(limit: int = 50) -> list[dict]: - """Get trending meme tokens across chains.""" - from app.unified_provider import get_unified_provider - - provider = get_unified_provider() - memes = [] - - for chain in MEME_CHAINS: - try: - # Get trending from DexScreener - trending = await provider.get_dexscreener_trending(chain) - - for token in trending[:20]: - # Classify as meme based on name/symbol - category = classify_meme_category( - token.get("baseToken", {}).get("symbol", ""), - token.get("baseToken", {}).get("name", ""), - ) - - if category != "other": - memes.append( - { - "address": token.get("baseToken", {}).get("address"), - "symbol": token.get("baseToken", {}).get("symbol"), - "name": token.get("baseToken", {}).get("name"), - "chain": chain, - "category": category, - "price_usd": token.get("priceUsd"), - "volume_24h": token.get("volume", {}).get("h24"), - "price_change_24h": token.get("priceChange", {}).get("h24"), - "liquidity_usd": token.get("liquidity", {}).get("usd"), - "fdv": token.get("fdv"), - "pair_age_hours": token.get("pairCreatedAt"), - "is_meme": True, - } - ) - except Exception as e: - logger.debug(f"Error getting meme trending for {chain}: {e}") - - # Sort by volume - memes.sort(key=lambda x: x.get("volume_24h", 0), reverse=True) - - return memes[:limit] - - -async def get_smart_money_in_memes(limit: int = 20) -> list[dict]: - """Track known smart money wallets trading memes.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Query for smart money activities in meme tokens - result = ( - supabase.table("smart_money_activities") - .select(""" - *, - wallets ( - wallet_address, - wallet_label, - wallet_category, - win_rate, - total_pnl_usd - ) - """) - .eq("is_meme", True) - .order("amount_usd", desc=True) - .limit(limit) - .execute() - ) - - return result.data or [] - - -async def get_meme_wins_losses(period: str = "24h") -> dict: - """Get biggest wins and losses in meme tokens.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Calculate time range - if period == "24h": - time_range = datetime.now(UTC) - timedelta(hours=24) - elif period == "7d": - time_range = datetime.now(UTC) - timedelta(days=7) - elif period == "30d": - time_range = datetime.now(UTC) - timedelta(days=30) - else: - time_range = datetime.now(UTC) - timedelta(hours=24) - - # Get biggest wins - wins = ( - supabase.table("whale_movements") - .select("*") - .gte("collected_at", time_range.isoformat()) - .eq("transaction_type", "sell") - .order("amount_usd", desc=True) - .limit(20) - .execute() - ) - - # Get biggest losses - losses = ( - supabase.table("whale_movements") - .select("*") - .gte("collected_at", time_range.isoformat()) - .eq("transaction_type", "buy") - .order("amount_usd", desc=True) - .limit(20) - .execute() - ) - - return { - "period": period, - "wins": wins.data or [], - "losses": losses.data or [], - } - - -def classify_meme_category(symbol: str, name: str) -> str: - """Classify meme token into category.""" - text = f"{symbol} {name}".lower() - - for category, keywords in MEME_CATEGORIES.items(): - if any(kw in text for kw in keywords): - return category - - return "other" - - -# ── KOL Intelligence ────────────────────────────────────────── - - -async def get_kol_scorecard(kol_handle: str) -> dict | None: - """Get KOL scorecard with track record.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Get KOL profile - result = supabase.table("kols").select("*").eq("twitter_handle", kol_handle).execute() - - if not result.data: - return None - - kol = result.data[0] - - # Get their past calls - calls = ( - supabase.table("kol_calls") - .select("*") - .eq("kol_id", kol["id"]) - .order("called_at", desc=True) - .limit(50) - .execute() - ) - - # Calculate stats - total_calls = len(calls.data) if calls.data else 0 - winning_calls = len([c for c in (calls.data or []) if c.get("pnl_pct", 0) > 0]) - win_rate = (winning_calls / total_calls * 100) if total_calls > 0 else 0 - - avg_pnl = sum([c.get("pnl_pct", 0) for c in (calls.data or [])]) / total_calls if total_calls > 0 else 0 - - return { - "kol": kol, - "stats": { - "total_calls": total_calls, - "winning_calls": winning_calls, - "win_rate": round(win_rate, 2), - "average_pnl": round(avg_pnl, 2), - "follower_count": kol.get("follower_count"), - "engagement_rate": kol.get("engagement_rate"), - }, - "recent_calls": calls.data[:10] if calls.data else [], - } - - -async def get_top_kols_by_category(category: str = "memes", limit: int = 20) -> list[dict]: - """Get top KOLs by category with scorecards.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - result = ( - supabase.table("kols") - .select("*") - .eq("primary_category", category) - .order("win_rate", desc=True) - .order("follower_count", desc=True) - .limit(limit) - .execute() - ) - - return result.data or [] - - -# ── Social Monitoring ───────────────────────────────────────── - - -async def monitor_social_sentiment(token_address: str) -> dict: - """Monitor social sentiment for a token.""" - # LunarCrush integration - try: - from app.lunarcrush_connector import get_lunarcrush_connector - - lc = get_lunarcrush_connector() - - sentiment = await lc.get_sentiment(token_address) - - return { - "token": token_address, - "social_volume": sentiment.get("social_volume"), - "social_dominance": sentiment.get("social_dominance"), - "sentiment_score": sentiment.get("sentiment_score"), - "mentions_24h": sentiment.get("mentions_24h"), - "mentions_change_24h": sentiment.get("mentions_change_24h"), - } - except Exception: - return {"error": "LunarCrush not available"} - - -async def get_viral_crypto_posts(hours: int = 24) -> list[dict]: - """Get viral crypto posts from X/Twitter.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - time_range = datetime.now(UTC) - timedelta(hours=hours) - - result = ( - supabase.table("viral_posts") - .select("*") - .gte("posted_at", time_range.isoformat()) - .order("engagement_score", desc=True) - .limit(50) - .execute() - ) - - return result.data or [] - - -# ── Hack/Drain Alerts ──────────────────────────────────────── - - -async def get_recent_hacks_drains(hours: int = 24) -> list[dict]: - """Get recent hacks and drains from monitoring.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - time_range = datetime.now(UTC) - timedelta(hours=hours) - - result = ( - supabase.table("security_alerts") - .select("*") - .gte("detected_at", time_range.isoformat()) - .in_("alert_type", ["hack", "drain", "exploit", "rugpull"]) - .order("amount_usd", desc=True) - .limit(50) - .execute() - ) - - return result.data or [] - - -async def format_hack_alert_for_social(hack_data: dict) -> dict: - """Format hack alert for X/Telegram posting.""" - return { - "x_post": f"""🚨 HACK ALERT 🚨 - -Protocol: {hack_data.get("protocol_name", "Unknown")} -Amount: ${hack_data.get("amount_usd", 0):,.0f} -Chain: {hack_data.get("chain", "Unknown")} -Type: {hack_data.get("attack_type", "Unknown")} - -{hack_data.get("description", "")[:200]} - -#CryptoSecurity #DeFi #HackAlert""", - "telegram_post": f"""🚨 *HACK ALERT* 🚨 - -*Protocol:* {hack_data.get("protocol_name", "Unknown")} -*Amount:* ${hack_data.get("amount_usd", 0):,.0f} -*Chain:* {hack_data.get("chain", "Unknown")} -*Type:* {hack_data.get("attack_type", "Unknown")} - -{hack_data.get("description", "")[:500]} - -Stay safe out there! 🔒""", - "severity": hack_data.get("severity", "medium"), - "amount_usd": hack_data.get("amount_usd", 0), - } - - -# ── Wallet Screenshot Generation ────────────────────────────── - - -async def generate_wallet_screenshot(wallet_address: str, pnl_data: dict) -> str: - """Generate wallet PnL screenshot for sharing.""" - # This would use a graphics API (Alibaba, etc.) to generate images - # For now, return placeholder - - return { - "wallet": wallet_address, - "total_pnl": pnl_data.get("total_pnl_usd", 0), - "win_rate": pnl_data.get("win_rate", 0), - "top_wins": pnl_data.get("top_wins", []), - "top_losses": pnl_data.get("top_losses", []), - "image_url": f"/api/v1/images/wallet/{wallet_address}/pnl-summary", - "share_url": f"https://rugmunch.io/wallet/{wallet_address}", - } - - -# ── Content Generation ──────────────────────────────────────── - - -async def generate_marketing_content(content_type: str, data: dict) -> dict: - """Generate marketing content using AI.""" - # This would call Alibaba's AI API for content generation - - templates = { - "win_announcement": """ -🎉 BIG WIN ALERT! 🎉 - -Wallet: {wallet_address[:8]}...{wallet_address[-6:]} -Token: {token_symbol} -Profit: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) - -This whale called it early and rode it all the way up! 🐋 - -Track smart money: https://rugmunch.io/wallet/{wallet_address} - -#Crypto #MemeCoin #SmartMoney -""", - "loss_announcement": """ -💀 LOSS PORN 💀 - -Wallet: {wallet_address[:8]}...{wallet_address[-6:]} -Token: {token_symbol} -Loss: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) - -Oof. Another reminder to take profits! 📉 - -Learn from their mistakes: https://rugmunch.io/wallet/{wallet_address} - -#Crypto #Trading #LossPorn -""", - "kol_scorecard": """ -📊 KOL SCORECARD: @{kol_handle} - -Win Rate: {win_rate:.1f}% -Total Calls: {total_calls} -Avg PnL: {avg_pnl:.1f}% -Followers: {follower_count:,} - -Track their calls: https://rugmunch.io/kol/{kol_handle} - -#CryptoTwitter #KOL #Alpha -""", - } - - template = templates.get(content_type, "") - - # Format template with data - content = template.format(**data) if template else "" - - return { - "content_type": content_type, - "x_post": content[:280], - "telegram_post": content, - "generated_at": datetime.now(UTC).isoformat(), - } +from app.domains.intelligence.meme_intelligence import * # noqa: F403 diff --git a/app/mount.py b/app/mount.py index 11095e2..8f93d55 100644 --- a/app/mount.py +++ b/app/mount.py @@ -36,7 +36,7 @@ ROUTER_MODULES: Final[list[str]] = [ "app.api.v1.admin.alerts_webhook", # /api/v1/admin/alerts/webhook "app.api.v1.admin.glitchtip_test", # /api/v1/_test/* (T07) "app.api.v1.catalog", # /api/v1/catalog/* - "app.api.v1.mcp", # /mcp/* (JSON-RPC + plain JSON) + "app.domains.mcp.router", # /mcp/* (JSON-RPC + plain JSON) # Domain facades (per v4.0 T28-T34) "app.domains.news", # /api/v1/news/* "app.domains.news.admin_router", # /api/v1/news/_admin/* diff --git a/app/n8n_intelligence_workflows.py b/app/n8n_intelligence_workflows.py index 763ec33..359e5cb 100644 --- a/app/n8n_intelligence_workflows.py +++ b/app/n8n_intelligence_workflows.py @@ -1,438 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.n8n_intelligence_workflows. + +Migrate imports from `app.n8n_intelligence_workflows` to `app.domains.intelligence.n8n_intelligence_workflows`. +This shim will be removed in Phase 3 cleanup. """ -n8n Workflow Automation - Market Intelligence & Premium Data Pulls. -Scheduled workflows for efficient data collection from multiple sources. -Runs every 30-60 minutes based on scan volume and data freshness needs. +from __future__ import annotations -Integrations: -- CoinGecko: trending, global metrics, top gainers/losers -- DexScreener: new pairs, trending tokens, volume spikes -- Dune Analytics: custom queries for whale tracking, scam patterns -- Helius: token mints, large transfers, new token deployments -- Moralis: EVM whale movements, new contract deployments -- Arkham: entity labeling, exchange flows -- Nansen: smart money tracking (if available) -- Forta: real-time threat detection -""" - -import logging -from datetime import UTC, datetime - -logger = logging.getLogger(__name__) - -# ── n8n Workflow Definitions ───────────────────────────────── - -N8N_BASE_URL = "https://n8n.rugmunch.io" -N8N_WEBHOOK_URL = f"{N8N_BASE_URL}/webhook" - -# Market Intelligence Workflows -MARKET_INTEL_WORKFLOWS = { - "trending_tokens": { - "name": "Trending Tokens Collector", - "schedule": "*/30 * * * *", # Every 30 minutes - "sources": ["coingecko", "dexscreener", "geckoterminal"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/trending", - "description": "Collect trending tokens from multiple DEXs", - "output_table": "market_trending_tokens", - }, - "whale_movements": { - "name": "Whale Movement Tracker", - "schedule": "*/15 * * * *", # Every 15 minutes (high priority) - "sources": ["helius", "moralis", "arkham"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/whales", - "description": "Track large transfers across chains", - "output_table": "whale_movements", - }, - "new_token_deployments": { - "name": "New Token Deployments", - "schedule": "*/10 * * * *", # Every 10 minutes (fast detection) - "sources": ["helius", "moralis", "dexscreener"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/new-tokens", - "description": "Detect new token deployments in real-time", - "output_table": "new_token_deployments", - }, - "volume_spikes": { - "name": "Volume Spike Detector", - "schedule": "*/5 * * * *", # Every 5 minutes (critical) - "sources": ["dexscreener", "geckoterminal", "birdeye"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/volume-spikes", - "description": "Detect unusual volume increases", - "output_table": "volume_spikes", - }, - "global_metrics": { - "name": "Market Global Metrics", - "schedule": "0 * * * *", # Every hour - "sources": ["coingecko"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/global", - "description": "Global crypto market metrics", - "output_table": "market_global_metrics", - }, - "defi_protocols": { - "name": "DeFi Protocol Analytics", - "schedule": "0 */2 * * *", # Every 2 hours - "sources": ["defillama", "dune"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/defi", - "description": "DeFi TVL, volume, user metrics", - "output_table": "defi_protocol_metrics", - }, - "nft_market": { - "name": "NFT Market Intelligence", - "schedule": "0 */4 * * *", # Every 4 hours - "sources": ["alchemy", "opensea_api"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/nft", - "description": "NFT floor prices, volume, trends", - "output_table": "nft_market_metrics", - }, -} - -# Premium Intelligence Workflows -PREMIUM_INTEL_WORKFLOWS = { - "smart_money_tracking": { - "name": "Smart Money Tracker", - "schedule": "*/30 * * * *", # Every 30 minutes - "sources": ["nansen", "arkham", "dune"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/smart-money", - "description": "Track known smart money wallets", - "output_table": "smart_money_activities", - "tier": "premium", - }, - "exchange_flows": { - "name": "Exchange Flow Analysis", - "schedule": "*/15 * * * *", # Every 15 minutes - "sources": ["arkham", "dune", "glassnode"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/exchange-flows", - "description": "Track exchange inflows/outflows", - "output_table": "exchange_flows", - "tier": "premium", - }, - "insider_trading": { - "name": "Insider Trading Detection", - "schedule": "*/20 * * * *", # Every 20 minutes - "sources": ["dune", "arkham", "helius"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/insider", - "description": "Detect potential insider trading patterns", - "output_table": "insider_trading_alerts", - "tier": "premium_plus", - }, - "launchpad_monitor": { - "name": "Launchpad Monitor", - "schedule": "*/10 * * * *", # Every 10 minutes - "sources": ["dexscreener", "pumpfun_api", "geckoterminal"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/launchpad", - "description": "Monitor new launches across platforms", - "output_table": "launchpad_monitoring", - "tier": "premium", - }, - "cluster_analysis": { - "name": "Cluster Pattern Analysis", - "schedule": "0 */2 * * *", # Every 2 hours - "sources": ["internal_clustering", "dune"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/clusters", - "description": "Deep cluster pattern analysis", - "output_table": "cluster_patterns", - "tier": "premium_plus", - }, -} - -# Security Intelligence Workflows -SECURITY_INTEL_WORKFLOWS = { - "scam_detection": { - "name": "Real-time Scam Detection", - "schedule": "*/5 * * * *", # Every 5 minutes (critical) - "sources": ["forta", "internal_ml", "community_reports"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/scams", - "description": "Detect new scam patterns", - "output_table": "scam_alerts", - "priority": "critical", - }, - "rugpull_detection": { - "name": "Rugpull Detection", - "schedule": "*/2 * * * *", # Every 2 minutes (ultra-critical) - "sources": ["internal_ml", "liquidity_monitor"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/rugpulls", - "description": "Detect rugpull patterns in real-time", - "output_table": "rugpull_alerts", - "priority": "critical", - }, - "contract_vulnerabilities": { - "name": "Contract Vulnerability Scanner", - "schedule": "0 * * * *", # Every hour - "sources": ["slither", "mythril", "internal_scanner"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/vulnerabilities", - "description": "Scan new contracts for vulnerabilities", - "output_table": "contract_vulnerabilities", - "priority": "high", - }, - "phishing_detection": { - "name": "Phishing Domain Detection", - "schedule": "0 */6 * * *", # Every 6 hours - "sources": ["guardian", "cryptoscamdb", "community"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/phishing", - "description": "Detect new phishing domains", - "output_table": "phishing_domains", - "priority": "high", - }, -} - - -# ── n8n Workflow Execution ──────────────────────────────────── - - -async def trigger_n8n_workflow(workflow_name: str, data: dict | None = None) -> bool: - """Trigger an n8n workflow via webhook.""" - import httpx - - # Find workflow config - all_workflows = { - **MARKET_INTEL_WORKFLOWS, - **PREMIUM_INTEL_WORKFLOWS, - **SECURITY_INTEL_WORKFLOWS, - } - workflow = all_workflows.get(workflow_name) - - if not workflow: - logger.error(f"Workflow not found: {workflow_name}") - return False - - webhook_url = workflow["endpoint"] - - try: - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.post( - webhook_url, - json={ - "workflow": workflow_name, - "triggered_at": datetime.now(UTC).isoformat(), - "data": data or {}, - }, - ) - - if response.status_code in (200, 201, 202): - logger.info(f"Triggered workflow: {workflow_name}") - return True - else: - logger.error(f"Workflow trigger failed: {workflow_name} - {response.status_code}") - return False - - except Exception as e: - logger.error(f"Workflow trigger error: {workflow_name} - {e}") - return False - - -async def execute_market_intelligence_pull(workflow_name: str): - """Execute a market intelligence data pull.""" - workflow = MARKET_INTEL_WORKFLOWS.get(workflow_name) - if not workflow: - return - - logger.info(f"Executing market intel pull: {workflow['name']}") - - # Collect data from sources - collected_data = { - "workflow": workflow_name, - "sources": workflow["sources"], - "collected_at": datetime.now(UTC).isoformat(), - "data": {}, - } - - # Source-specific collection logic - for source in workflow["sources"]: - try: - if source == "coingecko": - from app.coingecko_connector import get_coingecko_connector - - connector = get_coingecko_connector() - - if "trending" in workflow_name.lower(): - trending = await connector.get_trending() - collected_data["data"]["coingecko_trending"] = trending - elif "global" in workflow_name.lower(): - global_data = await connector.get_global_metrics() - collected_data["data"]["coingecko_global"] = global_data - - elif source == "dexscreener": - from app.unified_provider import get_unified_provider - - provider = get_unified_provider() - - if "trending" in workflow_name.lower(): - trending = await provider.get_dexscreener_trending() - collected_data["data"]["dexscreener_trending"] = trending - elif "volume" in workflow_name.lower(): - spikes = await provider.get_volume_spikes() - collected_data["data"]["dexscreener_spikes"] = spikes - - elif source == "helius": - from app.chain_client import get_chain_client - - client = get_chain_client() - - if "new" in workflow_name.lower(): - new_tokens = await client.get_new_token_mints(limit=50) - collected_data["data"]["helius_new_tokens"] = new_tokens - elif "whale" in workflow_name.lower(): - whales = await client.get_large_transfers(limit=20) - collected_data["data"]["helius_whales"] = whales - - except Exception as e: - logger.error(f"Error collecting from {source}: {e}") - collected_data["data"][source] = {"error": str(e)} - - # Store in Supabase - await _store_intelligence_data(workflow["output_table"], collected_data) - - # Trigger alerts if needed - await _process_intelligence_alerts(workflow_name, collected_data) - - -async def _store_intelligence_data(table_name: str, data: dict): - """Store intelligence data in Supabase.""" - try: - import os - - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Insert data - supabase.table(table_name).insert( - { - "data": data, - "collected_at": data.get("collected_at"), - "workflow": data.get("workflow"), - } - ).execute() - - logger.info(f"Stored intelligence data in {table_name}") - - except Exception as e: - logger.error(f"Failed to store intelligence data: {e}") - - -async def _process_intelligence_alerts(workflow_name: str, data: dict): - """Process intelligence data and trigger alerts.""" - # Check for significant findings - if "whale" in workflow_name.lower(): - # Check for unusually large transfers - whales = data.get("data", {}).get("helius_whales", []) - for whale in whales: - amount = whale.get("amount", 0) - if amount > 1000: # >1000 SOL - await _create_alert("whale_movement", whale) - - elif "scam" in workflow_name.lower() or "rugpull" in workflow_name.lower(): - # Immediate alert for security issues - await _create_alert("security_critical", data) - - elif "volume" in workflow_name.lower(): - # Check for unusual volume spikes - spikes = data.get("data", {}).get("dexscreener_spikes", []) - for spike in spikes: - if spike.get("volume_change_pct", 0) > 500: # >500% increase - await _create_alert("volume_spike", spike) - - -async def _create_alert(alert_type: str, data: dict): - """Create an intelligence alert.""" - try: - import os - - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - alert_data = { - "alert_type": alert_type, - "severity": "critical" if "security" in alert_type else "high", - "data": data, - "created_at": datetime.now(UTC).isoformat(), - "is_read": False, - } - - supabase.table("intelligence_alerts").insert(alert_data).execute() - - logger.info(f"Created intelligence alert: {alert_type}") - - except Exception as e: - logger.error(f"Failed to create alert: {e}") - - -# ── Dune Analytics Integration ──────────────────────────────── - - -async def execute_dune_query(query_id: str, params: dict | None = None) -> dict | None: - """Execute a Dune Analytics query.""" - import os - - import httpx - - dune_api_key = os.getenv("DUNE_API_KEY", "") - if not dune_api_key: - logger.warning("DUNE_API_KEY not configured") - return None - - try: - async with httpx.AsyncClient(timeout=60.0) as client: - # Execute query - response = await client.post( - f"https://api.dune.com/api/v1/query/{query_id}/execute", - headers={"X-Dune-API-Key": dune_api_key}, - json=params or {}, - ) - - if response.status_code != 200: - logger.error(f"Dune query failed: {response.status_code}") - return None - - execution_id = response.json().get("execution_id") - - # Wait for results - import asyncio - - for _ in range(10): # Max 10 attempts - await asyncio.sleep(2) - - result_response = await client.get( - f"https://api.dune.com/api/v1/execution/{execution_id}/results", - headers={"X-Dune-API-Key": dune_api_key}, - ) - - if result_response.status_code == 200: - result = result_response.json() - if result.get("state") == "QUERY_STATE_COMPLETED": - return result.get("result", {}).get("rows", []) - - return None - - except Exception as e: - logger.error(f"Dune query error: {e}") - return None - - -# Pre-configured Dune queries for RMI -DUNE_QUERIES = { - "ethereum_whale_transfers": { - "query_id": "1234567", # Replace with actual query ID - "description": "Track large ETH transfers from known whale wallets", - "schedule": "*/15 * * * *", - }, - "defi_protocol_volumes": { - "query_id": "2345678", - "description": "Daily DEX volumes across major protocols", - "schedule": "0 * * * *", - }, - "nft_wash_trading": { - "query_id": "3456789", - "description": "Detect potential NFT wash trading patterns", - "schedule": "0 */6 * * *", - }, - "stablecoin_flows": { - "query_id": "4567890", - "description": "Track USDC/USDT flows to/from exchanges", - "schedule": "*/30 * * * *", - }, - "new_contract_deployments": { - "query_id": "5678901", - "description": "New contract deployments with large funding", - "schedule": "*/10 * * * *", - }, -} +from app.domains.intelligence.n8n_intelligence_workflows import * # noqa: F403 diff --git a/app/news_intelligence.py b/app/news_intelligence.py index 2ef6565..de043ff 100644 --- a/app/news_intelligence.py +++ b/app/news_intelligence.py @@ -1,165 +1,8 @@ -#!/usr/bin/env python3 -""" -RMI News Intelligence v3 - Industry Best -========================================= -AI-powered news pipeline: categorization, sentiment, trending, briefing. -Uses MiniMax ($20/mo flat) + Ollama Cloud. +"""Deprecated shim - re-exports app.domains.intelligence.news_intelligence. + +Migrate imports from `app.news_intelligence` to `app.domains.intelligence.news_intelligence`. +This shim will be removed in Phase 3 cleanup. """ +from __future__ import annotations -import json -import logging -import os -import urllib.request -from collections import Counter -from datetime import UTC, datetime - -logger = logging.getLogger("rmi.news_v3") -OLLAMA_KEY = os.getenv("OLLAMA_API_KEY", os.getenv("DEEPSEEK_API_KEY", "")) -OLLAMA_URL = "https://ollama.com/v1/chat/completions" - -# Simple in-memory trending tracker -_trending_topics = Counter() -_breaking_alerts = [] - - -def analyze_article(title: str, content: str = "") -> dict: - """Full AI analysis of a news article.""" - text = f"{title} {content[:300]}" - - # Category (fast, cached) - from app.ai_pipeline_v3 import classify_news - - category = classify_news(title, content) - - # Sentiment via MiniMax (batched - 1 call per article is fine at flat rate) - sentiment = "neutral" - try: - k = os.getenv("OLLAMA_API_KEY", "") - if not k: - with open("/app/.env") as f: - for line in f: - if line.startswith("OLLAMA_API_KEY"): - k = line.strip().split("=", 1)[1] - break - if not k: - return {"category": category, "sentiment": "neutral", "is_breaking": False} - body = json.dumps( - { - "model": "deepseek-v4-flash", - "messages": [ - { - "role": "system", - "content": "Classify sentiment: BULLISH BEARISH NEUTRAL. Reply one word only.", - }, - {"role": "user", "content": text[:400]}, - ], - "max_tokens": 10, - "temperature": 0.1, - } - ).encode() - req = urllib.request.Request( - OLLAMA_URL, - data=body, - headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, - ) - resp = urllib.request.urlopen(req, timeout=8) - sentiment = json.loads(resp.read())["choices"][0]["message"]["content"].strip().upper() - if "BULL" in sentiment: - sentiment = "bullish" - elif "BEAR" in sentiment: - sentiment = "bearish" - else: - sentiment = "neutral" - except Exception as e: - logger.warning(f"Sentiment failed: {e}") - - # Track trending topics - for word in title.lower().split(): - if len(word) > 4 and word not in ( - "after", - "before", - "while", - "could", - "would", - "should", - "their", - "there", - "these", - "those", - "about", - "which", - ): - _trending_topics[word] += 1 - - # Detect breaking news - is_breaking = category == "SCAM" or any( - w in text.lower() for w in ["hacked", "exploited", "drained", "rug pulled", "emergency"] - ) - - return { - "category": category, - "sentiment": sentiment, - "is_breaking": is_breaking, - "analyzed_at": datetime.now(UTC).isoformat(), - } - - -def get_trending(limit: int = 10) -> list: - """Get trending topics from recent article analysis.""" - return [{"topic": word, "count": count} for word, count in _trending_topics.most_common(limit)] - - -def get_breaking() -> list: - """Get breaking news alerts.""" - return _breaking_alerts[-10:] - - -def daily_briefing() -> str: - """Generate an AI-powered daily news briefing using Ollama Cloud.""" - trending = get_trending(5) - topics = ", ".join(f"{t['topic']}({t['count']})" for t in trending) - - k = OLLAMA_KEY - body = json.dumps( - { - "model": "deepseek-v4-flash", - "messages": [ - { - "role": "system", - "content": "Write a 3-sentence daily crypto news briefing. Mention trending topics. Professional tone. Under 100 words.", - }, - {"role": "user", "content": f"Trending topics: {topics}"}, - ], - "max_tokens": 150, - "temperature": 0.5, - } - ).encode() - - try: - req = urllib.request.Request( - OLLAMA_URL, - data=body, - headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, - ) - resp = urllib.request.urlopen(req, timeout=15) - return json.loads(resp.read())["choices"][0]["message"]["content"].strip() - except Exception: - return f"Daily briefing: {topics} are trending in crypto news today." - - -def news_search(query: str, articles: list, limit: int = 10) -> list: - """Smart search across articles with relevance ranking.""" - results = [] - q_lower = query.lower() - for a in articles: - score = 0 - if q_lower in a.get("title", "").lower(): - score += 10 - if q_lower in a.get("content", "").lower(): - score += 5 - if q_lower in a.get("category", "").lower(): - score += 3 - if score > 0: - a["relevance"] = score - results.append(a) - return sorted(results, key=lambda x: x.get("relevance", 0), reverse=True)[:limit] +from app.domains.intelligence.news_intelligence import * # noqa: F403 diff --git a/app/prediction_market_service.py b/app/prediction_market_service.py index 0588baf..16ef225 100644 --- a/app/prediction_market_service.py +++ b/app/prediction_market_service.py @@ -1,1122 +1,13 @@ +"""Deprecated shim - re-exports app.domains.markets. + +Migrate imports from `app.prediction_market_service` to `app.domains.markets`. +This shim will be removed in Phase 3 cleanup. """ -RMI Prediction Market Intelligence Service -=========================================== -Multi-source prediction market data aggregation for crypto security intelligence. - -Data sources (all free, zero auth for read-only): - - Polymarket: Gamma API (search/discovery), CLOB API (prices/history), Data API (trades) - - Kalshi: REST API /series, /markets, /events, /orderbook (unauthenticated) - - Limitless: REST API /markets (Base chain, crypto-native) - - Manifold: REST API /markets (play-money, open-source sentiment signals) - -Open-source reference implementations (GitHub): - - homerun (braedonsaunders/homerun): Open-source prediction market platform for - Polymarket + Kalshi. Python strategies, backtesting, data sources, live trading. - - prediction-market-edge-bot: SX Bet + Polymarket aggregator with smart order routing. - - Awesome-Prediction-Market-Tools (aarora4): Curated directory of 50+ tools - including Oddpool (cross-venue aggregator), analytics dashboards, trading bots. - -Architecture: - - Direct external API calls (NEVER route through own API - anti-circular-dependency rule) - - All 4 sources queried in parallel with individual try/except - - Results normalized into unified PredictionMarket dataclass - - Redis caching: 30s TTL prices, 5min searches, 1hr digests - -Integration points: - - SENTINEL scanner: cross-reference token risk scores with market probability - - Wallet Memory Bank: entity/deployer reputation from prediction market odds - - RugMaps: visual correlation between market odds and on-chain wallet clusters - - x402 tools: expose as paid security intelligence endpoints - -Pitfalls: - - Polymarket Gamma API double-encodes outcomePrices/clobTokenIds as JSON strings - - One source timeout must not kill the entire call - individual try/except per source - - Prediction market data is probabilistic, not definitive - always cross-reference - with on-chain scanner results - - Don't poll every market every tick - use targeted search + category filters -""" - -import asyncio -import hashlib -import json -import logging -import os -from dataclasses import dataclass, field -from datetime import UTC, datetime - -import httpx - -logger = logging.getLogger("prediction_market") - -# ── API Endpoints ──────────────────────────────────────────────── - -POLYMARKET_GAMMA = "https://gamma-api.polymarket.com" -POLYMARKET_CLOB = "https://clob.polymarket.com" -POLYMARKET_DATA = "https://data-api.polymarket.com" - -KALSHI_BASE = "https://external-api.kalshi.com/trade-api/v2" - -LIMITLESS_BASE = "https://api.limitless.exchange" - -MANIFOLD_BASE = "https://api.manifold.markets/v0" - -# ── Category mappings for security relevance ──────────────────── - -SECURITY_KEYWORDS = [ - "hack", - "exploit", - "rug", - "scam", - "fraud", - "breach", - "leak", - "drain", - "phish", - "backdoor", - "vulnerability", - "zero-day", - "sanction", - "indict", - "arrest", - "freeze", - "seize", - "clampdown", - "depeg", - "insolvent", - "bankrupt", - "collapse", - "default", - "theft", - "heist", - "compromise", - "ransomware", - "malware", - "SEC", - "CFTC", - "DOJ", - "FBI", - "regulatory", - "enforcement", -] - -CRYPTO_KEYWORDS = [ - "bitcoin", - "ethereum", - "solana", - "crypto", - "defi", - "token", - "blockchain", - "web3", - "nft", - "stablecoin", - "usdt", - "usdc", - "dai", - "exchange", - "binance", - "coinbase", - "uniswap", - "aave", - "tether", - "circle", - "layer", - "L1", - "L2", - "rollup", - "bridge", - "polygon", - "arbitrum", - "optimism", - "avalanche", - "fantom", - "chainlink", - "makerdao", - "lido", - "eigenlayer", -] - -# ── Dataclasses ───────────────────────────────────────────────── - - -@dataclass -class PredictionMarket: - """Unified prediction market result across all sources.""" - - source: str # "polymarket" | "kalshi" | "limitless" | "manifold" - source_id: str # native ID from source (slug, ticker, etc.) - question: str - slug: str - probability_yes: float # 0.0-1.0 - probability_no: float # 0.0-1.0 - volume_usd: float - liquidity_usd: float = 0.0 - category: str = "" - tags: list[str] = field(default_factory=list) - tokens_mentioned: list[str] = field(default_factory=list) - is_security_relevant: bool = False - is_crypto_relevant: bool = False - url: str = "" - ends_at: str | None = None - updated_at: str = "" - - def __post_init__(self): - """Auto-classify relevance based on keywords in question.""" - q_lower = self.question.lower() - self.is_security_relevant = any(kw in q_lower for kw in SECURITY_KEYWORDS) - self.is_crypto_relevant = any(kw in q_lower for kw in CRYPTO_KEYWORDS) - - -@dataclass -class PredictionDigest: - """Daily intelligence digest of security-relevant prediction markets.""" - - generated_at: str - total_markets_searched: int - security_relevant_count: int - crypto_relevant_count: int - top_threats: list[PredictionMarket] = field(default_factory=list) - token_specific_markets: list[PredictionMarket] = field(default_factory=list) - ecosystem_risk_markets: list[PredictionMarket] = field(default_factory=list) - regulatory_markets: list[PredictionMarket] = field(default_factory=list) - - -# ── Service ───────────────────────────────────────────────────── - - -class PredictionMarketService: - """Multi-source prediction market data with parallel fetching + caching.""" - - def __init__(self, http_client: httpx.AsyncClient | None = None): - self._http = http_client or httpx.AsyncClient(timeout=15.0) - self._redis = None # Lazy init via get_redis() - - def _get_redis(self): - """Lazy Redis connection for caching. Returns None if unavailable.""" - if self._redis is not None: - return self._redis - try: - import redis.asyncio as aioredis - - self._redis = aioredis.from_url( - os.getenv("REDIS_URL", "redis://localhost:6379/0"), - decode_responses=True, - socket_connect_timeout=3, - ) - # Set to False if we couldn't actually connect - if self._redis is None: - self._redis = False - except Exception as e: - logger.warning(f"Redis unavailable, caching disabled: {e}") - self._redis = False - return self._redis if self._redis is not False else None - - # ── Public API ────────────────────────────────────────── - - async def search( - self, - query: str, - categories: list[str] | None = None, - min_volume: float = 0, - security_only: bool = False, - ) -> list[PredictionMarket]: - """Search all prediction market sources in parallel. - - Args: - query: Search term (token name, event, protocol, etc.) - categories: Optional filter by source categories - min_volume: Minimum USD volume to include - security_only: Only return security-relevant markets - """ - # Check Redis cache - cache_key = f"predmkt:search:{_cache_hash(query, categories, min_volume, security_only)}" - redis = self._get_redis() - if redis: - try: - cached = await redis.get(cache_key) - if cached: - markets_data = json.loads(cached) - return [_dict_to_market(d) for d in markets_data] - except Exception: - pass # Cache miss or Redis error - fall through to live query - - # Fire all 4 sources in parallel - results: list[list[PredictionMarket]] = await asyncio.gather( - self._search_polymarket(query), - self._search_kalshi(query), - self._search_limitless(query), - self._search_manifold(query), - return_exceptions=True, - ) - - # Flatten and handle exceptions - all_markets: list[PredictionMarket] = [] - sources = ["polymarket", "kalshi", "limitless", "manifold"] - for i, result in enumerate(results): - if isinstance(result, Exception): - logger.warning(f"{sources[i]} search failed: {result}") - continue - if isinstance(result, list): - all_markets.extend(result) - - # Filter - if min_volume > 0: - all_markets = [m for m in all_markets if m.volume_usd >= min_volume] - if security_only: - all_markets = [m for m in all_markets if m.is_security_relevant] - - # Sort by volume descending - all_markets.sort(key=lambda m: m.volume_usd, reverse=True) - - # Cache (5 min TTL) - if redis: - try: - await redis.setex(cache_key, 300, json.dumps([_market_to_dict(m) for m in all_markets[:50]])) - except Exception as e: - logger.warning(f"Redis cache write failed: {e}") - - return all_markets - - async def token_markets(self, symbol: str) -> list[PredictionMarket]: - """Find all prediction markets mentioning a specific token symbol.""" - results = await self.search(f'"{symbol}" token crypto', security_only=False) - - # Filter to markets actually about this token (mention in question) - symbol_lower = symbol.lower() - token_markets = [m for m in results if symbol_lower in m.question.lower()] - return token_markets - - async def security_digest(self) -> PredictionDigest: - """Generate daily intelligence digest of security-relevant prediction markets. - - Queries crypto + security keywords across all sources, categorizes - results by threat type: top threats, token-specific, ecosystem risk, - regulatory. - """ - cache_key = f"predmkt:digest:{datetime.now(UTC).strftime('%Y-%m-%d')}" - redis = self._get_redis() - if redis: - try: - cached = await redis.get(cache_key) - if cached: - data = json.loads(cached) - return _dict_to_digest(data) - except Exception: - pass # Cache miss or Redis error - fall through to live query - - # Search for security-relevant crypto markets - security_queries = [ - "crypto hack exploit scam", - "defi rug pull fraud", - "exchange insolvent breach", - "stablecoin depeg collapse", - "crypto regulation SEC enforcement", - "blockchain vulnerability zero-day", - ] - - all_markets: list[PredictionMarket] = [] - searches = [self.search(q, security_only=False) for q in security_queries] - results = await asyncio.gather(*searches, return_exceptions=True) - - for result in results: - if isinstance(result, list): - all_markets.extend(result) - - # De-duplicate by question similarity - seen_questions = set() - unique_markets = [] - for m in all_markets: - q_key = m.question.lower().strip()[:80] - if q_key not in seen_questions: - seen_questions.add(q_key) - unique_markets.append(m) - - # Categorize - top_threats = [m for m in unique_markets if m.is_security_relevant and m.volume_usd > 10000] - top_threats.sort(key=lambda m: m.volume_usd, reverse=True) - - token_specific = [ - m for m in unique_markets if m.is_crypto_relevant and m.is_security_relevant and len(m.tokens_mentioned) > 0 - ] - token_specific.sort(key=lambda m: m.volume_usd, reverse=True) - - ecosystem_risk = [ - m for m in unique_markets if m.is_crypto_relevant and not m.is_security_relevant and m.volume_usd > 50000 - ] - ecosystem_risk.sort(key=lambda m: m.volume_usd, reverse=True) - - regulatory = [ - m - for m in unique_markets - if m.is_security_relevant - and any(kw in m.question.lower() for kw in ["sec", "cftc", "doj", "regulation", "sanction", "ban"]) - ] - regulatory.sort(key=lambda m: m.volume_usd, reverse=True) - - digest = PredictionDigest( - generated_at=datetime.now(UTC).isoformat(), - total_markets_searched=len(unique_markets), - security_relevant_count=len([m for m in unique_markets if m.is_security_relevant]), - crypto_relevant_count=len([m for m in unique_markets if m.is_crypto_relevant]), - top_threats=top_threats[:20], - token_specific_markets=token_specific[:20], - ecosystem_risk_markets=ecosystem_risk[:10], - regulatory_markets=regulatory[:10], - ) - - # Cache (1 hour) - if redis: - try: - await redis.setex(cache_key, 3600, json.dumps(_digest_to_dict(digest))) - except Exception as e: - logger.warning(f"Redis digest cache write failed: {e}") - - return digest - - async def trending(self, limit: int = 20, source: str | None = None) -> list[PredictionMarket]: - """Get top trending prediction markets by volume across all sources.""" - # Fetch top events from each source in parallel - tasks = [] - if not source or source == "polymarket": - tasks.append(self._trending_polymarket(limit)) - else: - tasks.append(asyncio.sleep(0)) # placeholder - - if not source or source == "kalshi": - tasks.append(self._trending_kalshi(limit)) - else: - tasks.append(asyncio.sleep(0)) - - if not source or source == "limitless": - tasks.append(self._trending_limitless(limit)) - else: - tasks.append(asyncio.sleep(0)) - - results = await asyncio.gather(*tasks, return_exceptions=True) - - all_markets = [] - for result in results: - if isinstance(result, list): - all_markets.extend(result) - elif isinstance(result, Exception): - pass # Individual source failures logged in _trending_* methods - - all_markets.sort(key=lambda m: m.volume_usd, reverse=True) - return all_markets[:limit] - - async def market_detail(self, source: str, market_id: str) -> PredictionMarket | None: - """Get detailed data for a specific market including orderbook.""" - if source == "polymarket": - return await self._polymarket_detail(market_id) - elif source == "kalshi": - return await self._kalshi_detail(market_id) - # Limitless and Manifold details on demand - return None - - # ── Polymarket ────────────────────────────────────────── - - async def _search_polymarket(self, query: str) -> list[PredictionMarket]: - """Search Polymarket Gamma API.""" - try: - resp = await self._http.get( - f"{POLYMARKET_GAMMA}/public-search", - params={"q": query}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Polymarket search returned {resp.status_code}") - return [] - - data = resp.json() - events = data.get("events", []) - markets = [] - - for event in events[:10]: - for m in event.get("markets", [])[:5]: - pm = self._parse_polymarket_market(m, event) - if pm: - markets.append(pm) - - return markets - except Exception as e: - logger.warning(f"Polymarket search error: {e}") - return [] - - def _parse_polymarket_market(self, m: dict, event: dict | None = None) -> PredictionMarket | None: - """Parse a Polymarket market dict into unified PredictionMarket.""" - try: - question = m.get("question", "") - slug = m.get("slug", "") - - # Parse double-encoded JSON fields - prices = self._parse_json_field(m.get("outcomePrices", "[]")) - self._parse_json_field(m.get("outcomes", "[]")) - self._parse_json_field(m.get("clobTokenIds", "[]")) - - if isinstance(prices, list) and len(prices) >= 2: - prob_yes = float(prices[0]) - prob_no = float(prices[1]) - else: - prob_yes = 0.5 - prob_no = 0.5 - - volume = float(m.get("volume", 0)) - liquidity = float(m.get("liquidity", 0)) - - # Extract token mentions from question - tokens_mentioned = _extract_token_symbols(question) - - tags = [] - if event: - tags.extend([t.get("label", "") for t in event.get("tags", [])]) - - return PredictionMarket( - source="polymarket", - source_id=slug, - question=question, - slug=slug, - probability_yes=prob_yes, - probability_no=prob_no, - volume_usd=volume, - liquidity_usd=liquidity, - category=m.get("category", event.get("category", "") if event else ""), - tags=tags, - tokens_mentioned=tokens_mentioned, - url=f"https://polymarket.com/event/{slug}" if slug else "", - ends_at=m.get("endDate", event.get("endDate", "") if event else ""), - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Polymarket market: {e}") - return None - - async def _trending_polymarket(self, limit: int) -> list[PredictionMarket]: - """Get trending Polymarket events by volume.""" - try: - resp = await self._http.get( - f"{POLYMARKET_GAMMA}/events", - params={ - "limit": limit, - "active": "true", - "closed": "false", - "order": "volume", - "ascending": "false", - }, - timeout=10.0, - ) - if resp.status_code != 200: - return [] - - events = resp.json() - markets = [] - for event in events[:limit]: - for m in event.get("markets", [])[:3]: - pm = self._parse_polymarket_market(m, event) - if pm: - markets.append(pm) - return markets - except Exception as e: - logger.warning(f"Polymarket trending error: {e}") - return [] - - async def _polymarket_detail(self, slug: str) -> PredictionMarket | None: - """Get detailed Polymarket market data including CLOB prices.""" - try: - # Fetch from Gamma - resp = await self._http.get( - f"{POLYMARKET_GAMMA}/markets", - params={"slug": slug}, - timeout=10.0, - ) - if resp.status_code != 200: - return None - - data = resp.json() - if not data: - return None - - m = data[0] - pm = self._parse_polymarket_market(m) - - # Also fetch CLOB price for live data - if pm: - tokens = self._parse_json_field(m.get("clobTokenIds", "[]")) - if isinstance(tokens, list) and len(tokens) >= 2: - try: - price_resp = await self._http.get( - f"{POLYMARKET_CLOB}/price", - params={"token_id": tokens[0], "side": "buy"}, - timeout=5.0, - ) - if price_resp.status_code == 200: - price_data = price_resp.json() - live_price = float(price_data.get("price", pm.probability_yes)) - pm.probability_yes = live_price - pm.probability_no = 1.0 - live_price - except Exception: - pass # CLOB price is a bonus, Gamma price is fine - - return pm - except Exception as e: - logger.warning(f"Polymarket detail error for {slug}: {e}") - return None - - # ── Kalshi ────────────────────────────────────────────── - - async def _search_kalshi(self, query: str) -> list[PredictionMarket]: - """Search Kalshi by scanning events then fetching their markets.""" - try: - # Step 1: Get open events (organized by category, not sports-dominant) - resp = await self._http.get( - f"{KALSHI_BASE}/events", - params={"status": "open", "limit": 50}, - headers={"Accept": "application/json"}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Kalshi events returned {resp.status_code}") - return [] - - data = resp.json() - events = data.get("events", []) - query_lower = query.lower() - query_terms = query_lower.split() - - results = [] - - # Step 2: Check event titles for matches, then fetch markets - for i, event in enumerate(events[:10]): - event_title = event.get("title", "").lower() - event_ticker = event.get("ticker", "") - - # Match if query terms appear in event title - if not any(term in event_title for term in query_terms): - continue - - # Rate limit: small delay between event fetches - if i > 0: - await asyncio.sleep(0.3) - - # Step 3: Fetch markets for this event - try: - mr = await self._http.get( - f"{KALSHI_BASE}/markets", - params={"event_ticker": event_ticker, "status": "open", "limit": 10}, - headers={"Accept": "application/json"}, - timeout=8.0, - ) - if mr.status_code == 200: - markets_data = mr.json() - for m in markets_data.get("markets", []): - pm = self._parse_kalshi_market(m) - if pm: - results.append(pm) - except Exception: - continue - - return results - except Exception as e: - logger.warning(f"Kalshi search error: {e}") - return [] - - def _parse_kalshi_market(self, m: dict) -> PredictionMarket | None: - """Parse a Kalshi market dict into unified PredictionMarket.""" - try: - ticker = m.get("ticker", "") - title = m.get("title", "") - yes_bid = float(m.get("yes_bid_dollars", 0)) - volume = float(m.get("volume_fp", 0)) # Kalshi uses fake-penny notation - m.get("event_ticker", "") - category = m.get("category", "") - - # Skip multi-outcome markets (sports parlays, etc.) - they have no yes_bid - if yes_bid <= 0 or ",yes " in title.lower(): - return None - - prob_yes = yes_bid # Best YES bid approximates probability - prob_no = 1.0 - prob_yes if prob_yes else 0.5 - - tokens_mentioned = _extract_token_symbols(title) - - return PredictionMarket( - source="kalshi", - source_id=ticker, - question=title, - slug=ticker, - probability_yes=prob_yes, - probability_no=prob_no, - volume_usd=volume, - category=category, - tokens_mentioned=tokens_mentioned, - url=f"https://kalshi.com/markets/{ticker}" if ticker else "", - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Kalshi market: {e}") - return None - - async def _trending_kalshi(self, limit: int) -> list[PredictionMarket]: - """Get trending Kalshi markets by volume - uses events-first approach.""" - try: - # Get open events (avoid sports-multi-outcome noise from raw /markets) - resp = await self._http.get( - f"{KALSHI_BASE}/events", - params={"status": "open", "limit": min(limit * 2, 30)}, - timeout=10.0, - ) - if resp.status_code != 200: - return [] - - data = resp.json() - events = data.get("events", []) - - results = [] - for event in events[:limit]: - event_ticker = event.get("ticker", "") - try: - mr = await self._http.get( - f"{KALSHI_BASE}/markets", - params={"event_ticker": event_ticker, "status": "open", "limit": 5}, - timeout=8.0, - ) - if mr.status_code == 200: - markets_data = mr.json() - for m in markets_data.get("markets", []): - pm = self._parse_kalshi_market(m) - if pm: - results.append(pm) - except Exception: - continue - - return results[:limit] - except Exception as e: - logger.warning(f"Kalshi trending error: {e}") - return [] - - async def _kalshi_detail(self, ticker: str) -> PredictionMarket | None: - """Get detailed Kalshi market data including orderbook.""" - try: - resp = await self._http.get( - f"{KALSHI_BASE}/markets/{ticker}/orderbook", - timeout=10.0, - ) - if resp.status_code != 200: - return None - - data = resp.json() - orderbook = data.get("orderbook_fp", {}) - yes_bids = orderbook.get("yes_dollars", []) - best_yes = float(yes_bids[0][0]) if yes_bids else 0.5 - - # Also get market metadata - meta_resp = await self._http.get( - f"{KALSHI_BASE}/markets", - params={"ticker": ticker}, - timeout=10.0, - ) - if meta_resp.status_code == 200: - meta_data = meta_resp.json() - markets = meta_data.get("markets", []) - if markets: - pm = self._parse_kalshi_market(markets[0]) - if pm: - pm.probability_yes = best_yes - pm.probability_no = 1.0 - best_yes - return pm - - return None - except Exception as e: - logger.warning(f"Kalshi detail error for {ticker}: {e}") - return None - - # ── Limitless ─────────────────────────────────────────── - - async def _search_limitless(self, query: str) -> list[PredictionMarket]: - """Search Limitless Exchange markets by fetching active and filtering.""" - try: - resp = await self._http.get( - f"{LIMITLESS_BASE}/markets/active", - params={"limit": 25}, - headers={"Accept": "application/json"}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Limitless markets returned {resp.status_code}") - return [] - - data = resp.json() - all_markets = data.get("data", []) - - # Filter client-side by query terms - query_lower = query.lower() - query_terms = query_lower.split() - - results = [] - for m in all_markets: - title = m.get("title", "").lower() - if any(term in title for term in query_terms): - pm = self._parse_limitless_market(m) - if pm: - results.append(pm) - - return results - except Exception as e: - logger.warning(f"Limitless search error: {e}") - return [] - - def _parse_limitless_market(self, m: dict) -> PredictionMarket | None: - """Parse a Limitless market dict into unified PredictionMarket.""" - try: - title = m.get("title", "") - slug = m.get("slug", str(m.get("id", ""))) - - # Prices: [YES%, NO%] - e.g., [42.8, 57.2] - prices = m.get("prices", [50, 50]) - prob_yes = float(prices[0]) / 100 if isinstance(prices, list) and len(prices) >= 1 else 0.5 - prob_no = float(prices[1]) / 100 if isinstance(prices, list) and len(prices) >= 2 else 0.5 - - # Volume: use volumeFormatted if available, else volume - vol_str = m.get("volumeFormatted", str(m.get("volume", 0))) - volume = float(vol_str) if vol_str else 0.0 - - categories = m.get("categories", []) - category = categories[0] if categories else "" - tags = m.get("tags", []) - tokens_mentioned = _extract_token_symbols(title) - - return PredictionMarket( - source="limitless", - source_id=str(slug), - question=title, - slug=str(slug), - probability_yes=prob_yes, - probability_no=prob_no, - volume_usd=volume, - category=category, - tags=tags, - tokens_mentioned=tokens_mentioned, - url=f"https://limitless.exchange/markets/{slug}" if slug else "", - ends_at=m.get("expirationDate", ""), - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Limitless market: {e}") - return None - - async def _trending_limitless(self, limit: int) -> list[PredictionMarket]: - """Get trending Limitless markets.""" - try: - limit = min(limit, 25) # API max - resp = await self._http.get( - f"{LIMITLESS_BASE}/markets/active", - params={"limit": limit}, - headers={"Accept": "application/json"}, - timeout=10.0, - ) - if resp.status_code != 200: - return [] - - data = resp.json() - markets = data.get("data", []) - return [pm for m in markets[:limit] if (pm := self._parse_limitless_market(m))] - except Exception as e: - logger.warning(f"Limitless trending error: {e}") - return [] - - # ── Manifold ──────────────────────────────────────────── - - async def _search_manifold(self, query: str) -> list[PredictionMarket]: - """Search Manifold Markets (play-money, sentiment signals). - - Manifold is pure play-money but useful for: - - Forecasting community sentiment - - Early signal detection (top forecasters often move before real-money markets) - - Broad question coverage (more niche crypto questions than Polymarket) - """ - try: - resp = await self._http.get( - f"{MANIFOLD_BASE}/search-markets", - params={"term": query, "limit": 20}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Manifold search returned {resp.status_code}") - return [] - - data = resp.json() - contracts = data if isinstance(data, list) else data.get("contracts", data.get("markets", [])) - - results = [] - for c in contracts[:10]: - pm = self._parse_manifold_market(c) - if pm: - results.append(pm) - - return results - except Exception as e: - logger.warning(f"Manifold search error: {e}") - return [] - - def _parse_manifold_market(self, c: dict) -> PredictionMarket | None: - """Parse a Manifold contract into unified PredictionMarket.""" - try: - question = c.get("question", "") - slug = c.get("slug", c.get("id", "")) - prob = float(c.get("probability", c.get("prob", 0.5))) - volume = float(c.get("volume", c.get("volume24Hours", 0))) - - # Manifold uses "Mana" play money, volume is a signal but lower weight - tokens_mentioned = _extract_token_symbols(question) - tags = list(c.get("tags", [])) - - return PredictionMarket( - source="manifold", - source_id=str(slug), - question=question, - slug=str(slug), - probability_yes=prob, - probability_no=1.0 - prob, - volume_usd=volume, - category="", - tags=tags, - tokens_mentioned=tokens_mentioned, - url=f"https://manifold.markets/{c.get('creatorUsername', '')}/{slug}" if slug else "", - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Manifold market: {e}") - return None - - # ── Helpers ───────────────────────────────────────────── - - @staticmethod - def _parse_json_field(val): - """Parse double-encoded JSON fields (Polymarket Gamma API).""" - if isinstance(val, str): - try: - return json.loads(val) - except (json.JSONDecodeError, TypeError): - return val - return val - - -# ── Singleton ──────────────────────────────────────────────────── - -_service: PredictionMarketService | None = None - - -def get_prediction_market_service() -> PredictionMarketService: - """Get or create the singleton PredictionMarketService.""" - global _service - if _service is None: - _service = PredictionMarketService() - return _service - - -# ── Helpers: Token Extraction ──────────────────────────────────── - -# Common token symbols to detect in market questions -_COMMON_TOKENS = { - "BTC", - "ETH", - "SOL", - "USDT", - "USDC", - "DAI", - "BNB", - "XRP", - "ADA", - "DOGE", - "MATIC", - "POL", - "DOT", - "AVAX", - "LINK", - "UNI", - "AAVE", - "ARB", - "OP", - "SUI", - "APT", - "TIA", - "SEI", - "STRK", - "WLD", - "PEPE", - "SHIB", - "BONK", - "WIF", - "JUP", - "PYTH", - "RNDR", - "FET", - "AGIX", - "OCEAN", - "IMX", - "INJ", - "EIGEN", - "ENA", - "ETHFI", -} - -# Broader project names often referenced in prediction markets -_COMMON_PROJECTS = { - "polymarket", - "kalshi", - "manifold", - "uniswap", - "sushiswap", - "aave", - "compound", - "makerdao", - "maker", - "lido", - "eigenlayer", - "chainlink", - "arbitrum", - "optimism", - "polygon", - "avalanche", - "fantom", - "near", - "celestia", - "worldcoin", - "tether", - "circle", - "coinbase", - "binance", - "kraken", - "ftx", - "celcius", - "blockfi", - "three arrows", - "alameda", - "jump crypto", - "wintermute", - "curve", - "balancer", - "thorchain", - "osmosis", - "dydx", - "gmx", - "hyperliquid", - "jupiter", - "raydium", - "orca", - "wormhole", - "layerzero", - "zksync", - "starknet", - "scroll", - "linea", - "base", - "mantle", - "mode", - "blast", -} - - -def _extract_token_symbols(text: str) -> list[str]: - """Extract known token symbols and project names from text.""" - found = [] - text_upper = text.upper() - text_lower = text.lower() - - # Check token symbols (typically uppercase in text) - for token in _COMMON_TOKENS: - # Match as word boundary: " BTC " or "BTC's" or "$BTC" - if ( - f" {token} " in f" {text_upper} " - or f"${token}" in text_upper - or text_upper.startswith(f"{token} ") - or text_upper.endswith(f" {token}") - ) and token not in found: - found.append(token) - - # Check project names (case-insensitive) - for project in _COMMON_PROJECTS: - if project in text_lower and project.upper() not in found: - found.append(project) - - return found - - -# ── Caching Helpers ────────────────────────────────────────────── - - -def _cache_hash(*args) -> str: - """Create a short hash for cache keys.""" - raw = "|".join(str(a) for a in args) - return hashlib.md5(raw.encode()).hexdigest()[:12] - - -def _market_to_dict(m: PredictionMarket) -> dict: - """Serialize PredictionMarket to dict for JSON caching.""" - return { - "source": m.source, - "source_id": m.source_id, - "question": m.question, - "slug": m.slug, - "probability_yes": m.probability_yes, - "probability_no": m.probability_no, - "volume_usd": m.volume_usd, - "liquidity_usd": m.liquidity_usd, - "category": m.category, - "tags": m.tags, - "tokens_mentioned": m.tokens_mentioned, - "is_security_relevant": m.is_security_relevant, - "is_crypto_relevant": m.is_crypto_relevant, - "url": m.url, - "ends_at": m.ends_at, - "updated_at": m.updated_at, - } - - -def _dict_to_market(d: dict) -> PredictionMarket: - """Deserialize dict back to PredictionMarket.""" - return PredictionMarket( - source=d.get("source", ""), - source_id=d.get("source_id", ""), - question=d.get("question", ""), - slug=d.get("slug", ""), - probability_yes=float(d.get("probability_yes", 0.5)), - probability_no=float(d.get("probability_no", 0.5)), - volume_usd=float(d.get("volume_usd", 0)), - liquidity_usd=float(d.get("liquidity_usd", 0)), - category=d.get("category", ""), - tags=d.get("tags", []), - tokens_mentioned=d.get("tokens_mentioned", []), - is_security_relevant=d.get("is_security_relevant", False), - is_crypto_relevant=d.get("is_crypto_relevant", False), - url=d.get("url", ""), - ends_at=d.get("ends_at", ""), - updated_at=d.get("updated_at", ""), - ) - - -def _digest_to_dict(d: PredictionDigest) -> dict: - """Serialize PredictionDigest to dict for JSON caching.""" - return { - "generated_at": d.generated_at, - "total_markets_searched": d.total_markets_searched, - "security_relevant_count": d.security_relevant_count, - "crypto_relevant_count": d.crypto_relevant_count, - "top_threats": [_market_to_dict(m) for m in d.top_threats], - "token_specific_markets": [_market_to_dict(m) for m in d.token_specific_markets], - "ecosystem_risk_markets": [_market_to_dict(m) for m in d.ecosystem_risk_markets], - "regulatory_markets": [_market_to_dict(m) for m in d.regulatory_markets], - } - - -def _dict_to_digest(d: dict) -> PredictionDigest: - """Deserialize dict back to PredictionDigest.""" - return PredictionDigest( - generated_at=d.get("generated_at", ""), - total_markets_searched=d.get("total_markets_searched", 0), - security_relevant_count=d.get("security_relevant_count", 0), - crypto_relevant_count=d.get("crypto_relevant_count", 0), - top_threats=[_dict_to_market(m) for m in d.get("top_threats", [])], - token_specific_markets=[_dict_to_market(m) for m in d.get("token_specific_markets", [])], - ecosystem_risk_markets=[_dict_to_market(m) for m in d.get("ecosystem_risk_markets", [])], - regulatory_markets=[_dict_to_market(m) for m in d.get("regulatory_markets", [])], - ) +from __future__ import annotations + +from app.domains.markets import ( # noqa: F401 + PredictionDigest, + PredictionMarket, + PredictionMarketService, + get_prediction_market_service, +) diff --git a/app/routers/admin_backend.py b/app/routers/admin_backend.py index e5b662d..e197629 100644 --- a/app/routers/admin_backend.py +++ b/app/routers/admin_backend.py @@ -28,7 +28,7 @@ from datetime import datetime, timedelta from fastapi import APIRouter, Body, HTTPException, Request from pydantic import BaseModel, Field -from app.admin_backend import ( +from app.domains.admin import ( PERMISSIONS, AdminRole, AdminUserStore, diff --git a/app/routers/analytics.py b/app/routers/analytics.py index d8ff21c..6153f35 100644 --- a/app/routers/analytics.py +++ b/app/routers/analytics.py @@ -25,8 +25,8 @@ from typing import Any from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field -from app.admin_backend import AuditLogger, require_admin from app.analytics_engine import AnalyticsEngine, DashboardWidget, get_analytics_engine +from app.domains.admin import AuditLogger, require_admin logger = logging.getLogger("analytics_api") diff --git a/app/routers/bulletin_board.py b/app/routers/bulletin_board.py index f8341e8..afd96a2 100644 --- a/app/routers/bulletin_board.py +++ b/app/routers/bulletin_board.py @@ -25,8 +25,8 @@ Admin endpoints (require admin session): from fastapi import APIRouter, Body, HTTPException, Request from pydantic import BaseModel, Field -from app.admin_backend import AuditLogger, require_admin -from app.bulletin_board import ( +from app.domains.admin import AuditLogger, require_admin +from app.domains.bulletin import ( BulletinBoardManager, PostCategory, PostStatus, @@ -576,7 +576,7 @@ async def admin_moderate_comment( @router.get("/badges/{user_id}") async def get_badges(user_id: str): """Get user badges.""" - from app.bulletin_board import get_user_badges, get_user_reputation + from app.domains.bulletin import get_user_badges, get_user_reputation badges = await get_user_badges(user_id) rep = await get_user_reputation(user_id) @@ -613,7 +613,7 @@ async def x402_bot_post(request: Request, data: dict): Bot posting via x402 - $1 TO JOIN (one-time, 30-day membership). Pay $1 once, post unlimited for 30 days. Includes badges. """ - from app.bulletin_board import verify_x402_bot + from app.domains.bulletin import verify_x402_bot bot_addr = data.get("bot_address", "") tx_hash = data.get("tx_hash", "") diff --git a/app/routers/prediction_market_router.py b/app/routers/prediction_market_router.py index 37ebf7b..d5b4201 100644 --- a/app/routers/prediction_market_router.py +++ b/app/routers/prediction_market_router.py @@ -23,7 +23,7 @@ import logging from fastapi import APIRouter, HTTPException, Query -from app.prediction_market_service import ( +from app.domains.markets import ( PredictionDigest, PredictionMarket, get_prediction_market_service, diff --git a/app/routers/wallet_manager_v2.py b/app/routers/wallet_manager_v2.py index 869f5ef..c97ff16 100644 --- a/app/routers/wallet_manager_v2.py +++ b/app/routers/wallet_manager_v2.py @@ -33,7 +33,7 @@ from typing import Any from fastapi import APIRouter, Body, HTTPException, Request from pydantic import BaseModel, Field -from app.admin_backend import AuditLogger, require_admin +from app.domains.admin import AuditLogger, require_admin from app.wallet_manager_v2 import ( CHAIN_REGISTRY, PaymentRecord, diff --git a/app/routers/x402_mcp_handler.py b/app/routers/x402_mcp_handler.py index a63d55f..1832ac1 100644 --- a/app/routers/x402_mcp_handler.py +++ b/app/routers/x402_mcp_handler.py @@ -10,7 +10,7 @@ import os from fastapi import APIRouter, HTTPException, Request -from app.mcp.x402_mcp_server import TOOLS, handle_mcp_call +from app.domains.mcp.x402_mcp_server import TOOLS, handle_mcp_call router = APIRouter(tags=["x402-MCP"]) diff --git a/app/security_defense.py b/app/security_defense.py index 630d46b..c43f3e3 100644 --- a/app/security_defense.py +++ b/app/security_defense.py @@ -526,7 +526,7 @@ class HoneypotSystem: await BotDetectionEngine._log_security_event(event) # Auto-ban the IP - from app.admin_backend import SecurityManager + from app.domains.admin import SecurityManager await SecurityManager.block_ip(ip, f"Honeypot triggered: {path}", 168) # 7 days diff --git a/app/services/prediction_market_intel.py b/app/services/prediction_market_intel.py index a5c17e8..5cb4332 100644 --- a/app/services/prediction_market_intel.py +++ b/app/services/prediction_market_intel.py @@ -27,7 +27,7 @@ import logging from dataclasses import dataclass, field from datetime import UTC, datetime -from app.prediction_market_service import ( +from app.domains.markets import ( PredictionMarket, get_prediction_market_service, ) diff --git a/mypy-gate.ini b/mypy-gate.ini new file mode 100644 index 0000000..340ec5f --- /dev/null +++ b/mypy-gate.ini @@ -0,0 +1,14 @@ +[mypy] +strict = true +ignore_missing_imports = true +explicit_package_bases = true +exclude = (\.venv/|tests/|__pycache__/) + +[mypy-app.core.redis] +ignore_errors = false + +[mypy-app.domains.auth.*] +ignore_errors = false + +[mypy-*] +ignore_errors = true diff --git a/mypy.ini b/mypy.ini index efdbb65..8ae7d38 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,5 @@ [mypy] strict = true ignore_missing_imports = true +explicit_package_bases = true exclude = (\.venv/|tests/|__pycache__/) diff --git a/tests/unit/test_tier1_moat_mcp.py b/tests/unit/test_tier1_moat_mcp.py index 8675f50..b20363f 100644 --- a/tests/unit/test_tier1_moat_mcp.py +++ b/tests/unit/test_tier1_moat_mcp.py @@ -5,7 +5,7 @@ needing a live database or external service. They test the contract. """ from __future__ import annotations -from app.mcp.server import ( +from app.domains.mcp.server import ( MCP_PROTOCOL_VERSION, MCP_SERVER_VERSION, TOOL_CATALOG, diff --git a/tests/unit/test_tier2_eth_labels_mcp.py b/tests/unit/test_tier2_eth_labels_mcp.py index 0fcc24f..676b479 100644 --- a/tests/unit/test_tier2_eth_labels_mcp.py +++ b/tests/unit/test_tier2_eth_labels_mcp.py @@ -9,15 +9,15 @@ from unittest.mock import patch import pytest -from app.mcp.server import call_tool +from app.domains.mcp.server import call_tool @pytest.mark.asyncio async def test_eth_labels_query_tool_exists(): """Test that eth_labels_query tool is registered in the server.""" - import app.mcp.server + import app.domains.mcp.server # Check if it's in the catalog - tool_names = [tool["name"] for tool in app.mcp.server.TOOL_CATALOG] + tool_names = [tool["name"] for tool in app.domains.mcp.server.TOOL_CATALOG] assert "eth_labels_query" in tool_names print("✓ eth_labels_query found in tool catalog") @@ -25,8 +25,8 @@ async def test_eth_labels_query_tool_exists(): @pytest.mark.asyncio async def test_eth_labels_stats_tool_exists(): """Test that eth_labels_stats tool is registered in the server.""" - import app.mcp.server - tool_names = [tool["name"] for tool in app.mcp.server.TOOL_CATALOG] + import app.domains.mcp.server + tool_names = [tool["name"] for tool in app.domains.mcp.server.TOOL_CATALOG] assert "eth_labels_stats" in tool_names print("✓ eth_labels_stats found in tool catalog") @@ -34,7 +34,7 @@ async def test_eth_labels_stats_tool_exists(): @pytest.mark.asyncio async def test_eth_labels_query_version_tracked(): """Test that eth_labels_query tool has version tracking.""" - from app.mcp.server import TOOL_VERSIONS + from app.domains.mcp.server import TOOL_VERSIONS assert "eth_labels_query" in TOOL_VERSIONS version = TOOL_VERSIONS["eth_labels_query"] assert version == "1.0.0" # Version from Tier 2 @@ -44,7 +44,7 @@ async def test_eth_labels_query_version_tracked(): @pytest.mark.asyncio async def test_eth_labels_stats_version_tracked(): """Test that eth_labels_stats tool has version tracking.""" - from app.mcp.server import TOOL_VERSIONS + from app.domains.mcp.server import TOOL_VERSIONS assert "eth_labels_stats" in TOOL_VERSIONS version = TOOL_VERSIONS["eth_labels_stats"] assert version == "1.0.0" # Version from Tier 2 @@ -52,7 +52,7 @@ async def test_eth_labels_stats_version_tracked(): @pytest.mark.asyncio -@patch('app.mcp.tools.eth_labels_tool.query_eth_labels_db_mcp') +@patch('app.domains.mcp.tools.eth_labels_tool.query_eth_labels_db_mcp') async def test_eth_labels_query_calls_underlying_function(mock_query): """Test that eth_labels_query tool calls our implementation.""" # Mock the response @@ -71,7 +71,7 @@ async def test_eth_labels_query_calls_underlying_function(mock_query): @pytest.mark.asyncio -@patch('app.mcp.tools.eth_labels_tool.get_eth_labels_stats_mcp') +@patch('app.domains.mcp.tools.eth_labels_tool.get_eth_labels_stats_mcp') async def test_eth_labels_stats_calls_underlying_function(mock_stats): """Test that eth_labels_stats calls our implementation.""" mock_response = {"total_accounts": 106000, "tables": ["accounts"]}