From 757eefd227461ac25edca2c8209e8553886ad1dc Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 04:47:09 +0700 Subject: [PATCH 01/12] docs(state): update STATUS.md and ARCHITECTURE.md for Phase 4 progress --- ARCHITECTURE.md | 32 +++++++++++++++++++------------ STATUS.md | 51 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index be1609b..d339444 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,10 +1,11 @@ # RMI Backend — Architecture **Status:** canonical -**Last updated:** 2026-07-03 +**Last updated:** 2026-07-07 ## Overview FastAPI monolith serving crypto risk intelligence via HTTP, Telegram, and MCP. +Domains are consolidated under `app/domains/` per AUDIT-2026-Q3 Phase 4. ## High-level flow ``` @@ -12,9 +13,9 @@ Client (Web / Telegram / AI Agent) ↓ nginx → rmi-backend container (:8000) ↓ -FastAPI routers +FastAPI routers (mounted by app/mount.py) ↓ -Services / scanners / DataBus providers +app/domains//service.py → app/infra/ + app/domains//repository.py ↓ Postgres / Redis / Qdrant / Neo4j / ClickHouse ``` @@ -22,35 +23,42 @@ Postgres / Redis / Qdrant / Neo4j / ClickHouse ## Components | Path | Purpose | |---|---| -| `app/routers/` | HTTP route handlers (thin) | -| `app/scanners/` | Token risk scanners | -| `app/databus/` | 30+ third-party data providers | -| `app/telegram_bot/` | Telegram bot command handlers | +| `app/factory.py` | FastAPI app factory (lifespan, middleware, error handlers, routers) | +| `app/mount.py` | Single source of truth for router mounting | +| `app/core/` | Cross-cutting concerns (logging, errors, redis, http, auth, config) | +| `app/api/v1/` | Thin HTTP transport layer (public, auth, admin, x402, mcp) | +| `app/domains/` | Business logic domains: auth, billing, databus, news, reports, scanner, scanners, telegram, threat, token, tokens, wallet, x402 | +| `app/domains/scanners/` | Token risk scanners (IP — will move to rmi-ip in Phase 6) | +| `app/domains/databus/` | 30+ third-party data provider integrations | +| `app/domains/telegram/rugmunchbot/` | Telegram bot command handlers | | `app/mcp/` | MCP manifest and tool manager | -| `app/facilitators/` | x402 payment verification | -| `app/wallet_memory/` | Wallet label + clustering subsystem | +| `app/infra/` | External integrations (ollama, langfuse, vector stores, chain clients) | +| `app/wallet_memory/` | Wallet label + clustering subsystem (IP — will move to rmi-ip in Phase 6) | +| `app/routers/` | Legacy HTTP route handlers (being migrated to app/domains/ or app/api/v1/) | ## Storage | Store | Use | Container / Host | |---|---|---| | Postgres | Primary OLTP, Ponder indexer data | `rmi-postgres` | | Redis | Cache, Celery queue | `rmi-redis` | -| Qdrant | Vector embeddings | `rmi-qdrant` (down) | +| Qdrant | Vector embeddings | `rmi-qdrant` | | Neo4j | Wallet graph / labels | `rmi-neo4j` | | ClickHouse | Analytics, 39M address labels | `rmi-clickhouse` | ## Interfaces | Endpoint | Purpose | |---|---| -| `GET /health` | Health check (currently unhealthy due to Redis) | +| `GET /health` | Health check (degraded — ETH RPC endpoint missing) | +| `GET /metrics` | Prometheus metrics | | `GET /docs` | Swagger/OpenAPI docs | | `POST /scanner/scan` | Live token scan | | `POST /mcp/...` | MCP tool endpoints | ## Current issues -- `x402_tools.py` (212 KB) and `x402_enforcement.py` (108 KB) violate the 500-line limit and should become separate services or split by domain. +- `app/routers/x402_tools.py` (212 KB) and `app/routers/x402_enforcement.py` (108 KB) violate the 500-line limit and should become separate services or split by domain. - `sqlite3` is used in 5 files for local state; should migrate to Postgres/aiosqlite. - F-string logs and `print()` statements prevent structured log aggregation. +- `mypy` has a structural issue with `app/domains/auth/` that needs resolution. ## Deployment ```bash diff --git a/STATUS.md b/STATUS.md index 0377364..5272908 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,7 +1,7 @@ # STATUS -- rmi-backend **Owner:** Rug Munch Media LLC Engineering -**Last updated:** 2026-07-06 -- Phase 1 of [AUDIT-2026-Q3.md](AUDIT-2026-Q3.md) complete. +**Last updated:** 2026-07-07 -- Phase 4 domain consolidation in progress; P4.1-P4.8 + P5.1 complete. ## What This Repo Is @@ -13,7 +13,7 @@ Stack: Python 3.12 / FastAPI / SQLAlchemy / async / Playwright / Pydantic v2 / R ## Current Status -**Phase 1 (development hygiene) complete.** Phase 2 (delete dead code) starts next per [AUDIT-2026-Q3.md](AUDIT-2026-Q3.md). +**Phase 4 (domain consolidation) is active.** Foundation (Phase 1) is solid, dead-router wiring (Phase 2) is partial, god-file splitting (Phase 3) is deferred, and standardization (Phase 5) has begun with CI trustworthiness and mypy config cleanup. ### Phase 1 -- DONE @@ -30,16 +30,44 @@ Committed to `main`: - `92a01ff` + `5294983` -- `AppError` hierarchy in `app/core/errors.py`, wired through `error_handlers.py` - `cd02714` + `2fb571e` -- author amend (canonical identity: `cryptorugmunch `) -### What Phase 1 DID NOT do (deferred) +### Phase 2 -- IN PROGRESS (partial) + +Rather than deleting 148 routers, we validated and mounted the useful ones in `app/mount.py` Wave 1-3 (2026-07-07). Dead routers remain in tree and will be archived in a later pass. + +### Phase 3 -- NOT STARTED + +God-files (`x402_tools.py`, `x402_enforcement.py`, `auth.py`, `databus/providers.py`, `telegram_bot/bot.py`, `wallet_manager_v2.py`) still await splitting. + +### Phase 4 -- IN PROGRESS + +Domain consolidation (AUDIT-2026-Q3 P4.x) — committed to `main`: +- `dca458e` -- P4.1 billing (`app/billing/` + `app/facilitators/` → `app/domains/billing/`) +- `7109a16` -- P4.2 wallet (`app/wallet/` → `app/domains/wallet/`) +- `948e41c` -- P4.3 auth (`app/auth/` → `app/domains/auth/`) +- `cab9740` -- P4.4 tokens (`app/tokens/` → `app/domains/tokens/`) +- `ed5b830` -- P4.5 databus (`providers/` + `_generated/` → `app/domains/databus/`) +- `56075cf` -- P4.6 telegram (`rugmunchbot/` → `app/domains/telegram/rugmunchbot/`) +- `3b7ef42` -- P4.7 rename `app/domain/` → `app/domains/` + consolidate +- `7cced4e` + `4686cb3` -- P4.8 scanners (`app/scanners/` → `app/domains/scanners/`) + +Remaining Phase 4: +- Finish databus root migration (`app/databus/` engine → `app/domains/databus/`) +- Finish telegram migration (`app/telegram_bot/` → `app/domains/telegram/`) +- Clean up deprecated shim dirs (`app/auth/`, `app/wallet/`, `app/billing/`, `app/facilitators/`, `app/scanners/`, `app/domain/`) +- Consolidate bulletin, intelligence, markets, admin, referral, MCP domains + +### Phase 5 -- STARTED + +- `d666ad2` -- P5.1 `HEALTH_CHECK_DURATION` + `test_factory_has_minimum_routes` +- `4686cb3` -- mypy config fixes (`mypy.ini` regex, `disallow_any_explicit` typo, `app.domains.*` module pattern) +- Still open: alembic, real coverage gate, structured logging, Sentry wiring, Prometheus middleware + +### What Phase 1-4 DID NOT do (deferred) - ~2K ruff warnings in legacy code (Phase 5) -- Real coverage 8.17% -> 80% gate (Phase 5, slowest part) -- `mypy.ini` line 8 parse error + `disallow_any_express_imports` typo (Phase 5) +- Real coverage 8.17% -> 80% gate (Phase 5) - Alembic migrations (still MISSING; Phase 5) -- 285 dead modules (Phase 2) -- 148 unmounted routers (Phase 2) - God-files split (Phase 3) -- Domain consolidation (Phase 4) ### Open Issues @@ -68,8 +96,11 @@ e404e90 feat(rmi-backend,audit): add app/main.py entrypoint delegating to factor - `gitleaks detect` reports 28 findings -- all **public ERC-20/Solana token contract addresses** (USDC, WETH, MATIC, BONK, DAI, plus cryptoscam-address dataset). NOT secrets. Suppressed via `.gitleaksignore` + `.gitleaks.toml` `[allowlist]`. No rotation needed. - All secrets live in gopass under `rmi/contacts/admin_recovery` etc. -## Next: Phase 2 (week 2) +## Next: Phase 4 completion -Archive **148 unmounted routers** + **~140 dead `app/*.py` files** to `app/_archive/legacy_2026_07/`. Goal: ~30% LOC reduction without changing any current behavior. +1. Finish `app/databus/` engine migration to `app/domains/databus/`. +2. Finish `app/telegram_bot/` migration to `app/domains/telegram/rugmunchbot/`. +3. Remove deprecated shim directories (`app/auth/`, `app/wallet/`, `app/billing/`, `app/facilitators/`, `app/scanners/`, `app/domain/`). +4. Consolidate bulletin, intelligence, markets, admin, referral, MCP into `app/domains/`. See [AUDIT-2026-Q3.md](AUDIT-2026-Q3.md) for the full 6-phase plan. From 7bd204f9f5276afbf0b398284f534e59fe2c5914 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 04:50:09 +0700 Subject: [PATCH 02/12] refactor(domains): remove deprecated shim dirs + update remaining legacy imports (P4 cleanup) --- app/_archive/legacy_2026_07/mcp_server.py | 4 +- app/billing/__init__.py | 23 --------- app/domain/__init__.py | 25 --------- app/facilitators/__init__.py | 33 ------------ app/routers/mcp_server.py | 32 ++++++++---- app/routers/x402_enforcement.py | 63 ++++++++++++----------- app/routers/x402_tools.py | 8 +-- app/scanners/__init__.py | 43 ---------------- 8 files changed, 61 insertions(+), 170 deletions(-) delete mode 100644 app/billing/__init__.py delete mode 100644 app/domain/__init__.py delete mode 100644 app/facilitators/__init__.py delete mode 100644 app/scanners/__init__.py diff --git a/app/_archive/legacy_2026_07/mcp_server.py b/app/_archive/legacy_2026_07/mcp_server.py index 0d28b81..a72dca7 100644 --- a/app/_archive/legacy_2026_07/mcp_server.py +++ b/app/_archive/legacy_2026_07/mcp_server.py @@ -46,7 +46,7 @@ def _get_tools() -> dict[str, Any]: def _get_facilitator_count() -> int: try: - from app.facilitators.base import get_registry + from app.domains.billing.facilitators.base import get_registry return len(get_registry().get_all()) except Exception: @@ -57,7 +57,7 @@ def _desc(tool_id: str, pricing: dict) -> str: d = pricing.get("description", "") if d and d != tool_id and len(d) > 10: return d - FALLBACKS = { + fallbacks = { "airdrop_check": "Verify airdrop legitimacy -- contract audit, distribution analysis, scam pattern detection. Know if an airdrop is real or a wallet drainer before connecting.", "airdrop_finder": "Discover active and upcoming airdrops across all major chains. Eligibility checks, value estimation, claim deadlines, and Sybil detection.", "all_in_one": "All-in-One Audit -- comprehensive security scan: rug pull, honeypot, clone detection, contract audit, and ownership analysis in a single call.", diff --git a/app/billing/__init__.py b/app/billing/__init__.py deleted file mode 100644 index 1fa389a..0000000 --- a/app/billing/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -"""billing — DEPRECATED shim. Use app.domains.billing. - -Phase 4.1 of AUDIT-2026-Q3.md moved app/billing/ to app/domains/billing/. -This shim re-exports the canonical surface and aliases submodules so that -`from app.billing.x402.router import router` keeps working. - -MIGRATION: replace `from app.billing.x402 import ...` with -`from app.domains.billing.x402 import ...`. -""" -import sys as _sys -from app.domains.billing import x402 as _x402 -from app.domains.billing.x402 import router as _router - -# Public re-exports -from app.domains.billing.x402 import ( # noqa: F401 - X402Enforcer, - TOOL_PRICES, - CHAIN_USDC, -) - -# Alias submodules so `from app.billing.x402.router import router` works -_sys.modules["app.billing.x402"] = _x402 -_sys.modules["app.billing.x402.router"] = _router diff --git a/app/domain/__init__.py b/app/domain/__init__.py deleted file mode 100644 index ea0f690..0000000 --- a/app/domain/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""domain - DEPRECATED shim. Use app.domains. - -Phase 4.7 of AUDIT-2026-Q3.md renamed app/domain/ to app/domains/. -This shim re-exports for legacy callers. -""" -import sys as _sys -import importlib as _importlib - -import app.domains as _new -_sys.modules['app.domain'] = _new - -_sys.modules['app.domain.alerts'] = _importlib.import_module('app.domains.alerts') -_sys.modules['app.domain.auth'] = _importlib.import_module('app.domains.auth') -_sys.modules['app.domain.billing'] = _importlib.import_module('app.domains.billing') -_sys.modules['app.domain.databus'] = _importlib.import_module('app.domains.databus') -_sys.modules['app.domain.labels'] = _importlib.import_module('app.domains.labels') -_sys.modules['app.domain.news'] = _importlib.import_module('app.domains.news') -_sys.modules['app.domain.reports'] = _importlib.import_module('app.domains.reports') -_sys.modules['app.domain.scanner'] = _importlib.import_module('app.domains.scanner') -_sys.modules['app.domain.telegram'] = _importlib.import_module('app.domains.telegram') -_sys.modules['app.domain.threat'] = _importlib.import_module('app.domains.threat') -_sys.modules['app.domain.token'] = _importlib.import_module('app.domains.token') -_sys.modules['app.domain.tokens'] = _importlib.import_module('app.domains.tokens') -_sys.modules['app.domain.wallet'] = _importlib.import_module('app.domains.wallet') -_sys.modules['app.domain.x402'] = _importlib.import_module('app.domains.x402') diff --git a/app/facilitators/__init__.py b/app/facilitators/__init__.py deleted file mode 100644 index 1c391f2..0000000 --- a/app/facilitators/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -"""facilitators — DEPRECATED shim. Use app.domains.billing.facilitators. - -Phase 4.1 of AUDIT-2026-Q3.md moved app/facilitators/ to -app/domains/billing/facilitators/. This shim re-exports the public surface -and aliases submodules so legacy imports like -`from app.facilitators.base import Facilitator` keep working. - -MIGRATION: replace `from app.facilitators.base import ...` with -`from app.domains.billing.facilitators.base import ...`. -""" -import sys as _sys -import importlib as _importlib - -# Re-export public surface -from app.domains.billing.facilitators import ( # noqa: F401 - Facilitator, - FacilitatorRegistry, - SettlementResult, - VerificationResult, - FacilitatorRouter, - get_facilitator_router, -) - -# Alias submodules -for _name in ( - "base", "config", "router", - "asterpay", "bitcoin_selfverify", "cloudflare_x402", "coinbase_cdp", - "eip7702", "merx_tron", "payai", "pieverse", "primev", - "satoshi", "startup", "tron_selfverify", "x402_rs", -): - _sys.modules[f"app.facilitators.{_name}"] = _importlib.import_module( - f"app.domains.billing.facilitators.{_name}" - ) diff --git a/app/routers/mcp_server.py b/app/routers/mcp_server.py index 0d28b81..5971f42 100644 --- a/app/routers/mcp_server.py +++ b/app/routers/mcp_server.py @@ -46,7 +46,7 @@ def _get_tools() -> dict[str, Any]: def _get_facilitator_count() -> int: try: - from app.facilitators.base import get_registry + from app.domains.billing.facilitators.base import get_registry return len(get_registry().get_all()) except Exception: @@ -57,7 +57,7 @@ def _desc(tool_id: str, pricing: dict) -> str: d = pricing.get("description", "") if d and d != tool_id and len(d) > 10: return d - FALLBACKS = { + fallbacks = { "airdrop_check": "Verify airdrop legitimacy -- contract audit, distribution analysis, scam pattern detection. Know if an airdrop is real or a wallet drainer before connecting.", "airdrop_finder": "Discover active and upcoming airdrops across all major chains. Eligibility checks, value estimation, claim deadlines, and Sybil detection.", "all_in_one": "All-in-One Audit -- comprehensive security scan: rug pull, honeypot, clone detection, contract audit, and ownership analysis in a single call.", @@ -138,7 +138,7 @@ def _desc(tool_id: str, pricing: dict) -> str: "whale_accumulation": "Whale accumulation monitor -- detect when large wallets are building positions, track accumulation rate and entry timing.", # noqa: F601 "whale_profile": "Whale profile -- deep analysis of a whale wallet: strategy classification, historical performance, holdings breakdown, and influence.", # noqa: F601 } - return FALLBACKS.get( + return fallbacks.get( tool_id, f"{tool_id.replace('_', ' ').title()} -- real-time crypto intelligence and security analysis.", ) @@ -482,7 +482,11 @@ def _build_tools_list(): cat = pricing.get("category", "analysis") cats[cat] = cats.get(cat, 0) + 1 real_tool = TOOL_ALIASES.get(tool_id, tool_id) - endpoint = f"/api/v1/x402-databus/{tool_id}" if tool_id in databus_ids else f"/api/v1/x402-tools/{real_tool}" + endpoint = ( + f"/api/v1/x402-databus/{tool_id}" + if tool_id in databus_ids + else f"/api/v1/x402-tools/{real_tool}" + ) tools[tool_id] = { "name": tool_id, # MCP spec: name is the tool ID "description": _desc(tool_id, pricing), @@ -492,7 +496,9 @@ def _build_tools_list(): "trial_free": int(pricing.get("trial_free", 1)), "chains": sorted(chains_data.keys()), "endpoint": endpoint, - "method": pricing.get("method", "POST") if isinstance(pricing.get("method"), str) else "POST", + "method": pricing.get("method", "POST") + if isinstance(pricing.get("method"), str) + else "POST", "databus": tool_id in databus_ids, } @@ -540,7 +546,7 @@ async def mcp_tools_list(request: Request): "trials": remaining, } except Exception: - pass + logger.debug("trial info fetch failed", exc_info=True) return { "server": f"{SERVER_NAME} MCP v{SERVER_VERSION}", @@ -844,7 +850,9 @@ async def mcp_jsonrpc(request: Request): } ) result = ( - resp.json() if "application/json" in (resp.headers.get("content-type", "")) else {"data": resp.text} + resp.json() + if "application/json" in (resp.headers.get("content-type", "")) + else {"data": resp.text} ) return JSONResponse( { @@ -915,7 +923,11 @@ async def mcp_call_tool(tool_id: str, request: Request): ) try: - body = await request.json() if request.headers.get("content-type") == "application/json" else {} + body = ( + await request.json() + if request.headers.get("content-type") == "application/json" + else {} + ) except Exception: body = {} @@ -952,7 +964,9 @@ async def mcp_call_tool(tool_id: str, request: Request): async with httpx.AsyncClient(timeout=45) as client: resp = await client.post(url, json=body, headers=headers) result = ( - resp.json() if "application/json" in (resp.headers.get("content-type", "")) else {"data": resp.text} + resp.json() + if "application/json" in (resp.headers.get("content-type", "")) + else {"data": resp.text} ) rh = {} for h in ( diff --git a/app/routers/x402_enforcement.py b/app/routers/x402_enforcement.py index 343a150..08d511a 100755 --- a/app/routers/x402_enforcement.py +++ b/app/routers/x402_enforcement.py @@ -1,41 +1,42 @@ -"""Backward-compat shim — moved to app.billing.x402.enforcement in P3B.""" -from app.billing.x402.enforcement import * # noqa: F401,F403 -from app.billing.x402.enforcement import ( # noqa: F401 - X402Enforcer, - TOOL_PRICES, +"""Backward-compat shim — moved to app.domains.billing.x402.enforcement in P3B.""" + +from app.domains.billing.x402.enforcement import * # noqa: F403 +from app.domains.billing.x402.enforcement import ( # noqa: F401 CHAIN_USDC, + EVM_PAY_TO, SECURITY_HEADERS, - check_idempotency, - check_trial, - consume_trial, - build_402_response, - parse_x_pay_header, - verify_payment, - verify_payment_via_router, - self_verify_evm_usdc, - x402_enforcement_middleware, - x402_discovery, - get_trial_status, - get_revenue, - get_pricing_suggestions, - request_refund, - _ensure_tool_prices, + SOL_PAY_TO, + TOOL_PRICES, + VERIFIER, + X402Enforcer, + _build_accepts_list, + _build_bazaar_extension, + _build_discovery_response, + _build_extra, + _build_resource_info, + _detect_token_from_asset, _ensure_pay_addresses, + _ensure_tool_prices, _ensure_verifier, _get_pay_to_address, _load_tool_prices, - _build_accepts_list, - _build_bazaar_extension, - _build_resource_info, - _build_discovery_response, - _resolve_pay_to, - _resolve_asset, - _build_extra, _record_refundable_payment, + _resolve_asset, + _resolve_pay_to, _verify_refund_ownership, - _detect_token_from_asset, + build_402_response, + check_idempotency, + check_trial, + consume_trial, + get_pricing_suggestions, get_redis_async, - EVM_PAY_TO, - SOL_PAY_TO, - VERIFIER, + get_revenue, + get_trial_status, + parse_x_pay_header, + request_refund, + self_verify_evm_usdc, + verify_payment, + verify_payment_via_router, + x402_discovery, + x402_enforcement_middleware, ) diff --git a/app/routers/x402_tools.py b/app/routers/x402_tools.py index a696339..2a57f98 100755 --- a/app/routers/x402_tools.py +++ b/app/routers/x402_tools.py @@ -1,11 +1,11 @@ -"""x402_tools.py — DEPRECATED shim. Re-exports from app.billing.x402. +"""x402_tools.py — DEPRECATED shim. Re-exports from app.domains.billing.x402. Phase 3A of AUDIT-2026-Q3.md split this 5,780-LOC god-file into focused modules under app/billing/x402/. This file is kept as a thin re-export shim so any caller that does `from app.routers.x402_tools import router` still works. MIGRATION: replace `from app.routers.x402_tools import router` with -`from app.billing.x402.router import router`. The legacy import path +`from app.domains.billing.x402.router import router`. The legacy import path will be removed in a future release. Re-exported public surface (preserves every symbol other modules import): @@ -25,8 +25,8 @@ Pre-existing issues surfaced (not introduced) by this refactor: home. Track: fix(f821). """ -from app.billing.x402.router import router -from app.billing.x402.shared import ( +from app.domains.billing.x402.router import router +from app.domains.billing.x402.shared import ( BUNDLES, HUMAN_PAY_TO, HUMAN_PAYMENT_TOKENS, diff --git a/app/scanners/__init__.py b/app/scanners/__init__.py deleted file mode 100644 index 68c27bd..0000000 --- a/app/scanners/__init__.py +++ /dev/null @@ -1,43 +0,0 @@ -"""scanners - DEPRECATED shim. Use app.domains.scanners. - -Phase 4.8 of AUDIT-2026-Q3.md moved app/scanners/ to app/domains/scanners/. -The 33 detection modules are IP and will be split to rmi-ip in Phase 6. - -This shim aliases all submodules so legacy imports like -`from app.scanners.bundle_detector import *` keep working. -""" -import sys as _sys -import importlib as _importlib - -_sys.modules["app.scanners.address_labeler"] = _importlib.import_module("app.domains.scanners.address_labeler") -_sys.modules["app.scanners.block_zero_sniper"] = _importlib.import_module("app.domains.scanners.block_zero_sniper") -_sys.modules["app.scanners.bundle_detector"] = _importlib.import_module("app.domains.scanners.bundle_detector") -_sys.modules["app.scanners.bytecode_similarity"] = _importlib.import_module("app.domains.scanners.bytecode_similarity") -_sys.modules["app.scanners.contract_authority"] = _importlib.import_module("app.domains.scanners.contract_authority") -_sys.modules["app.scanners.contract_diff"] = _importlib.import_module("app.domains.scanners.contract_diff") -_sys.modules["app.scanners.decompiler_analyzer"] = _importlib.import_module("app.domains.scanners.decompiler_analyzer") -_sys.modules["app.scanners.dev_reputation"] = _importlib.import_module("app.domains.scanners.dev_reputation") -_sys.modules["app.scanners.exchange_funder"] = _importlib.import_module("app.domains.scanners.exchange_funder") -_sys.modules["app.scanners.flash_loan_detector"] = _importlib.import_module("app.domains.scanners.flash_loan_detector") -_sys.modules["app.scanners.fund_flow_visualizer"] = _importlib.import_module("app.domains.scanners.fund_flow_visualizer") -_sys.modules["app.scanners.gas_trace"] = _importlib.import_module("app.domains.scanners.gas_trace") -_sys.modules["app.scanners.governance_attack"] = _importlib.import_module("app.domains.scanners.governance_attack") -_sys.modules["app.scanners.guilt_association"] = _importlib.import_module("app.domains.scanners.guilt_association") -_sys.modules["app.scanners.holder_analyzer"] = _importlib.import_module("app.domains.scanners.holder_analyzer") -_sys.modules["app.scanners.honeypot_detector"] = _importlib.import_module("app.domains.scanners.honeypot_detector") -_sys.modules["app.scanners.liquidity_verifier"] = _importlib.import_module("app.domains.scanners.liquidity_verifier") -_sys.modules["app.scanners.metadata_fingerprint"] = _importlib.import_module("app.domains.scanners.metadata_fingerprint") -_sys.modules["app.scanners.mev_detector"] = _importlib.import_module("app.domains.scanners.mev_detector") -_sys.modules["app.scanners.oracle_manipulation"] = _importlib.import_module("app.domains.scanners.oracle_manipulation") -_sys.modules["app.scanners.proxy_detector"] = _importlib.import_module("app.domains.scanners.proxy_detector") -_sys.modules["app.scanners.pump_dump_detector"] = _importlib.import_module("app.domains.scanners.pump_dump_detector") -_sys.modules["app.scanners.pumpfun_analyzer"] = _importlib.import_module("app.domains.scanners.pumpfun_analyzer") -_sys.modules["app.scanners.pumpfun_enhanced"] = _importlib.import_module("app.domains.scanners.pumpfun_enhanced") -_sys.modules["app.scanners.rag_citations"] = _importlib.import_module("app.domains.scanners.rag_citations") -_sys.modules["app.scanners.sentiment_analyzer"] = _importlib.import_module("app.domains.scanners.sentiment_analyzer") -_sys.modules["app.scanners.sentinel_pipeline"] = _importlib.import_module("app.domains.scanners.sentinel_pipeline") -_sys.modules["app.scanners.sleep_cycle_scanner"] = _importlib.import_module("app.domains.scanners.sleep_cycle_scanner") -_sys.modules["app.scanners.social_signals"] = _importlib.import_module("app.domains.scanners.social_signals") -_sys.modules["app.scanners.social_velocity"] = _importlib.import_module("app.domains.scanners.social_velocity") -_sys.modules["app.scanners.static_analyzer"] = _importlib.import_module("app.domains.scanners.static_analyzer") -_sys.modules["app.scanners.wash_trading"] = _importlib.import_module("app.domains.scanners.wash_trading") From 59d6f2f0f6f27e83fd5143bd6bf7a70f2d87e0f6 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 04:56:23 +0700 Subject: [PATCH 03/12] refactor(auth): move JWT helpers to app.domains.auth.jwt, break auth circular import (P4.3 cont) --- app/domains/auth/__init__.py | 56 ++++++----------------------- app/domains/auth/jwt.py | 69 ++++++++++++++++++++++++++++++------ app/domains/auth/wallet.py | 4 +-- 3 files changed, 71 insertions(+), 58 deletions(-) diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index 8862117..230e026 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -14,10 +14,18 @@ from typing import Any import bcrypt from fastapi import APIRouter, HTTPException, Request -from jose import JWTError, jwt from pydantic import BaseModel, Field from app.core.redis import get_redis +from app.domains.auth.jwt import ( + JWT_ALGORITHM, + JWT_EXPIRY_DAYS, + JWT_SECRET, + _create_jwt, + _derive_user_id, + _verify_jwt, + generate_nonce, +) logger = logging.getLogger(__name__) @@ -95,9 +103,6 @@ def _verify_backup_code(code: str, hashed: str) -> bool: # ── Config ── -JWT_SECRET = os.getenv("JWT_SECRET", "rmi-jwt-secret-change-me") -JWT_ALGORITHM = "HS256" -JWT_EXPIRY_DAYS = 7 def hash_password(password: str) -> str: @@ -113,7 +118,7 @@ def verify_password(password: str, hashed: str) -> bool: return False -# ── Redis User Store ── +# ── Storage (Redis-backed user store) ── def _get_user(user_id: str) -> dict[str, Any] | None: """Fetch user record by id from Redis hash rmi:users.""" r = get_redis() @@ -159,45 +164,6 @@ def _is_valid_email(email: str) -> bool: return re.match(pattern, email) is not None -def _create_jwt(user_id: str, email: str, tier: str = "FREE", role: str = "USER", wallet: str | None = None) -> str: - now = datetime.utcnow() - payload = { - "sub": user_id, - "email": email, - "tier": tier, - "role": role, - "iat": now, - "exp": now + timedelta(days=JWT_EXPIRY_DAYS), - } - if wallet: - payload["wallet"] = wallet - return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) - - -def _verify_jwt(token: str) -> dict[str, Any] | None: - try: - payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) - return { - "id": payload["sub"], - "email": payload["email"], - "tier": payload.get("tier", "FREE"), - "role": payload.get("role", "USER"), - "wallet": payload.get("wallet"), - } - except JWTError: - return None - - -def _derive_user_id(email: str) -> str: - import hashlib - - return hashlib.sha256(email.lower().encode()).hexdigest()[:32] - - -def generate_nonce() -> str: - return secrets.token_urlsafe(16) - - # ── Storage (Redis-backed user store) ── class EmailLoginRequest(BaseModel): email: str @@ -438,7 +404,7 @@ async def wallet_nonce(req: WalletNonceRequest): @router.post("/wallet/verify", response_model=WalletAuthResponse) async def wallet_verify(req: WalletVerifyRequest): """Verify wallet signature and create session.""" - from app.auth_wallet import get_or_create_wallet_user, verify_wallet_signature + from app.domains.auth.wallet import get_or_create_wallet_user, verify_wallet_signature if not verify_wallet_signature(req.message, req.signature, req.address): raise HTTPException(status_code=401, detail="Invalid signature") diff --git a/app/domains/auth/jwt.py b/app/domains/auth/jwt.py index 80a85a5..6e83524 100644 --- a/app/domains/auth/jwt.py +++ b/app/domains/auth/jwt.py @@ -1,18 +1,65 @@ -"""JWT helpers - re-exports from app.auth for cleaner import paths. +"""JWT helpers. Phase 3B of AUDIT-2026-Q3.md. Public API: - JWT_SECRET, JWT_ALGORITHM, JWT_EXPIRY_DAYS - _create_jwt, _verify_jwt -- generate_nonce +- generate_nonce, _derive_user_id """ -from app.auth import ( # noqa: F401 - JWT_ALGORITHM, - JWT_EXPIRY_DAYS, - JWT_SECRET, - _create_jwt, - _derive_user_id, - _verify_jwt, - generate_nonce, -) +from __future__ import annotations + +import hashlib +import os +import secrets +from datetime import datetime, timedelta +from typing import Any + +from jose import JWTError, jwt + +JWT_SECRET = os.getenv("JWT_SECRET", "rmi-jwt-secret-change-me") +JWT_ALGORITHM = "HS256" +JWT_EXPIRY_DAYS = 7 + + +def _create_jwt( + user_id: str, + email: str, + tier: str = "FREE", + role: str = "USER", + wallet: str | None = None, +) -> str: + now = datetime.utcnow() + payload = { + "sub": user_id, + "email": email, + "tier": tier, + "role": role, + "iat": now, + "exp": now + timedelta(days=JWT_EXPIRY_DAYS), + } + if wallet: + payload["wallet"] = wallet + return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) + + +def _verify_jwt(token: str) -> dict[str, Any] | None: + try: + payload = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) + return { + "id": payload["sub"], + "email": payload["email"], + "tier": payload.get("tier", "FREE"), + "role": payload.get("role", "USER"), + "wallet": payload.get("wallet"), + } + except JWTError: + return None + + +def _derive_user_id(email: str) -> str: + return hashlib.sha256(email.lower().encode()).hexdigest()[:32] + + +def generate_nonce() -> str: + return secrets.token_urlsafe(16) diff --git a/app/domains/auth/wallet.py b/app/domains/auth/wallet.py index 3f82520..b4b222b 100644 --- a/app/domains/auth/wallet.py +++ b/app/domains/auth/wallet.py @@ -118,7 +118,7 @@ async def get_or_create_wallet_user(address: str, chain: str = "base") -> dict[s r.hset("rmi:users", user_id, json.dumps(user)) # Generate JWT token - from app.auth import _create_jwt + from app.domains.auth.jwt import _create_jwt # Use email if available, otherwise derive from address email = user.get("email") or f"{address.lower()}@wallet.rmi" @@ -145,7 +145,7 @@ async def verify_auth_token(token: str) -> dict[str, Any] | None: Returns: Dict with user info if valid, None if invalid/expired """ - from app.auth import _verify_jwt + from app.domains.auth.jwt import _verify_jwt try: user = _verify_jwt(token) From 3eb22495b19ab178755a2617d46e70b1de78d3bd Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 04:58:06 +0700 Subject: [PATCH 04/12] refactor(auth): move password helpers to app.domains.auth.passwords (P4.3 cont) --- app/domains/auth/__init__.py | 81 ++++++++++++++++++++++------------- app/domains/auth/jwt.py | 1 + app/domains/auth/passwords.py | 23 +++++++--- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index 230e026..742872a 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -12,20 +12,20 @@ import secrets from datetime import datetime, timedelta from typing import Any -import bcrypt from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel, Field from app.core.redis import get_redis from app.domains.auth.jwt import ( JWT_ALGORITHM, - JWT_EXPIRY_DAYS, + JWT_EXPIRY_DAYS as JWT_EXPIRY_DAYS, JWT_SECRET, _create_jwt, _derive_user_id, _verify_jwt, generate_nonce, ) +from app.domains.auth.passwords import hash_password, verify_password logger = logging.getLogger(__name__) @@ -105,19 +105,6 @@ def _verify_backup_code(code: str, hashed: str) -> bool: # ── Config ── -def hash_password(password: str) -> str: - """Hash password using bcrypt directly.""" - return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - - -def verify_password(password: str, hashed: str) -> bool: - """Verify password against hash.""" - try: - return bcrypt.checkpw(password.encode("utf-8"), hashed.encode("utf-8")) - except Exception: - return False - - # ── Storage (Redis-backed user store) ── def _get_user(user_id: str) -> dict[str, Any] | None: """Fetch user record by id from Redis hash rmi:users.""" @@ -464,7 +451,9 @@ async def get_current_user(request: Request): async def google_auth_url(): """Get Google OAuth URL.""" client_id = os.getenv("GOOGLE_CLIENT_ID") - redirect_uri = os.getenv("GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback") + redirect_uri = os.getenv( + "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" + ) if not client_id: return {"url": "/auth/callback?error=google_not_configured"} @@ -504,7 +493,9 @@ async def google_callback(request: Request): client_id = os.getenv("GOOGLE_CLIENT_ID") client_secret = os.getenv("GOOGLE_CLIENT_SECRET") - redirect_uri = os.getenv("GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback") + redirect_uri = os.getenv( + "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" + ) if not client_id or not client_secret: return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") @@ -556,7 +547,9 @@ async def google_callback(request: Request): r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) - jwt_token = _create_jwt(user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER")) + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) # Redirect to frontend with token return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=google") @@ -575,7 +568,9 @@ async def google_callback_post(request: Request): client_id = os.getenv("GOOGLE_CLIENT_ID") client_secret = os.getenv("GOOGLE_CLIENT_SECRET") - redirect_uri = os.getenv("GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback") + redirect_uri = os.getenv( + "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" + ) if not client_id or not client_secret: raise HTTPException(status_code=500, detail="Google OAuth not configured") @@ -627,7 +622,9 @@ async def google_callback_post(request: Request): r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) - jwt_token = _create_jwt(user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER")) + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) return { "access_token": jwt_token, @@ -680,7 +677,11 @@ async def telegram_auth(req: TelegramAuthRequest): user = _get_user(telegram_user_id) if not user: - display_name = f"{req.first_name or ''} {req.last_name or ''}".strip() or req.username or f"User{req.id}" + display_name = ( + f"{req.first_name or ''} {req.last_name or ''}".strip() + or req.username + or f"User{req.id}" + ) email = f"{req.id}@telegram.rmi" user = { @@ -702,7 +703,9 @@ async def telegram_auth(req: TelegramAuthRequest): r.hset("rmi:users", telegram_user_id, json.dumps(user)) r.hset("rmi:users:telegram", str(req.id), telegram_user_id) - jwt_token = _create_jwt(user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER")) + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) return { "access_token": jwt_token, @@ -725,7 +728,9 @@ async def telegram_auth(req: TelegramAuthRequest): async def github_auth_url(): """Get GitHub OAuth URL.""" client_id = os.getenv("GITHUB_CLIENT_ID") - redirect_uri = os.getenv("GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback") + redirect_uri = os.getenv( + "GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback" + ) if not client_id: return {"url": "/auth/callback?error=github_not_configured"} @@ -759,7 +764,9 @@ async def github_callback(request: Request): client_id = os.getenv("GITHUB_CLIENT_ID") client_secret = os.getenv("GITHUB_CLIENT_SECRET") - redirect_uri = os.getenv("GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback") + redirect_uri = os.getenv( + "GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback" + ) if not client_id or not client_secret: return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") @@ -802,7 +809,11 @@ async def github_callback(request: Request): if resp.status_code == 200: emails = resp.json() primary = next((e for e in emails if e.get("primary")), None) - email = primary.get("email") if primary else (emails[0].get("email") if emails else None) + email = ( + primary.get("email") + if primary + else (emails[0].get("email") if emails else None) + ) if not email: email = f"{github_user.get('login', 'github_user')}@github.rmi" @@ -827,7 +838,9 @@ async def github_callback(request: Request): r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) - jwt_token = _create_jwt(user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER")) + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=github") @@ -928,7 +941,9 @@ async def x_callback(request: Request): r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) - jwt_token = _create_jwt(user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER")) + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=x") @@ -971,7 +986,9 @@ async def twofa_setup(request: Request): # Check if already enabled if user.get("totp_secret"): - raise HTTPException(status_code=400, detail="2FA already enabled. Disable first to reconfigure.") + raise HTTPException( + status_code=400, detail="2FA already enabled. Disable first to reconfigure." + ) secret = _generate_totp_secret() uri = _get_totp_uri(secret, user["email"]) @@ -1015,7 +1032,9 @@ async def twofa_enable(req: TwoFAEnableRequest, request: Request): # Verify the code if not _verify_totp(secret, req.code): - raise HTTPException(status_code=400, detail="Invalid TOTP code. Check your authenticator app time sync.") + raise HTTPException( + status_code=400, detail="Invalid TOTP code. Check your authenticator app time sync." + ) # Enable 2FA on user record user["totp_secret"] = secret @@ -1138,7 +1157,9 @@ async def twofa_login(req: TwoFALoginRequest): # Check if 2FA is enabled if not user.get("totp_enabled") or not user.get("totp_secret"): - raise HTTPException(status_code=400, detail="2FA not enabled for this account. Use /login instead.") + raise HTTPException( + status_code=400, detail="2FA not enabled for this account. Use /login instead." + ) code = req.code.strip().replace("-", "") diff --git a/app/domains/auth/jwt.py b/app/domains/auth/jwt.py index 6e83524..6d63428 100644 --- a/app/domains/auth/jwt.py +++ b/app/domains/auth/jwt.py @@ -7,6 +7,7 @@ Public API: - _create_jwt, _verify_jwt - generate_nonce, _derive_user_id """ + from __future__ import annotations import hashlib diff --git a/app/domains/auth/passwords.py b/app/domains/auth/passwords.py index fb80810..cff223f 100644 --- a/app/domains/auth/passwords.py +++ b/app/domains/auth/passwords.py @@ -1,11 +1,24 @@ -"""Password helpers - re-exports from app.domains.auth. +"""Password helpers. Phase 3B of AUDIT-2026-Q3.md. Public API: - hash_password, verify_password """ -from app.auth import ( # noqa: F401 - hash_password, - verify_password, -) + +from __future__ import annotations + +import bcrypt + + +def hash_password(password: str) -> str: + """Hash password using bcrypt directly.""" + return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + + +def verify_password(password: str, hashed: str) -> bool: + """Verify password against hash.""" + try: + return bcrypt.checkpw(password.encode("utf-8"), hashed.encode("utf-8")) + except Exception: + return False From b31b564f363c94756eb5b4bf1dd222c8371665f8 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 13:05:25 +0700 Subject: [PATCH 05/12] refactor(auth): split store, schemas, deps, totp out of auth __init__.py (P4.3 cont) --- app/domains/auth/__init__.py | 311 +++++------------------------------ app/domains/auth/deps.py | 63 ++++++- app/domains/auth/schemas.py | 127 +++++++++++--- app/domains/auth/store.py | 71 +++++++- app/domains/auth/totp.py | 106 +++++++++--- 5 files changed, 353 insertions(+), 325 deletions(-) diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index 742872a..25cb745 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -2,20 +2,19 @@ Auth Router - Complete authentication system (email, wallet, OAuth, Telegram) """ -import base64 -import io import json import logging import os -import re -import secrets from datetime import datetime, timedelta -from typing import Any from fastapi import APIRouter, HTTPException, Request -from pydantic import BaseModel, Field from app.core.redis import get_redis +from app.domains.auth.deps import ( + get_current_user, + require_auth as require_auth, + require_public_profile as require_public_profile, +) from app.domains.auth.jwt import ( JWT_ALGORITHM, JWT_EXPIRY_DAYS as JWT_EXPIRY_DAYS, @@ -26,227 +25,53 @@ from app.domains.auth.jwt import ( generate_nonce, ) from app.domains.auth.passwords import hash_password, verify_password +from app.domains.auth.schemas import ( + EmailLoginRequest, + EmailRegisterRequest, + GoogleAuthResponse, + NonceResponse, + TelegramAuthRequest, + TwoFAEnableRequest, + TwoFALoginRequest, + TwoFASetupResponse as TwoFASetupResponse, + TwoFAStatusResponse, + TwoFAVerifyRequest, + UserResponse, + WalletAuthResponse, + WalletNonceRequest, + WalletVerifyRequest, +) +from app.domains.auth.store import ( + _delete_user as _delete_user, + _get_user, + _get_user_by_email, + _is_valid_email, + _save_user, +) +from app.domains.auth.totp import ( + PYOTP_AVAILABLE, + TOTP_DIGITS as TOTP_DIGITS, + TOTP_INTERVAL as TOTP_INTERVAL, + TOTP_ISSUER as TOTP_ISSUER, + TOTP_WINDOW as TOTP_WINDOW, + _build_totp as _build_totp, + _generate_backup_codes, + _generate_qr_base64, + _generate_totp_secret, + _get_totp_uri, + _hash_backup_code, + _verify_backup_code, + _verify_totp, +) logger = logging.getLogger(__name__) router = APIRouter(tags=["auth"]) -# ── 2FA / TOTP ── -try: - import pyotp - import qrcode - - PYOTP_AVAILABLE = True -except ImportError: - PYOTP_AVAILABLE = False - logger.warning("pyotp not installed - 2FA endpoints will return 503") - -TOTP_ISSUER = os.getenv("TOTP_ISSUER", "RugMunch Intelligence") -TOTP_DIGITS = 6 -TOTP_INTERVAL = 30 -TOTP_WINDOW = 1 # Allow ±1 interval clock drift - - -def _generate_totp_secret() -> str: - """Generate a new base32 TOTP secret.""" - if not PYOTP_AVAILABLE: - raise RuntimeError("pyotp not installed") - return pyotp.random_base32() - - -def _build_totp(secret: str) -> Any: - """Build a TOTP object from secret.""" - return pyotp.TOTP(secret, digits=TOTP_DIGITS, interval=TOTP_INTERVAL) - - -def _verify_totp(secret: str, code: str) -> bool: - """Verify a TOTP code with clock-drift tolerance.""" - if not PYOTP_AVAILABLE or not secret or not code: - return False - totp = _build_totp(secret) - return totp.verify(code, valid_window=TOTP_WINDOW) - - -def _get_totp_uri(secret: str, email: str) -> str: - """Get otpauth:// URI for QR code generation.""" - totp = _build_totp(secret) - return totp.provisioning_uri(name=email, issuer_name=TOTP_ISSUER) - - -def _generate_qr_base64(uri: str) -> str: - """Generate QR code as base64 PNG data URI.""" - img = qrcode.make(uri) - buf = io.BytesIO() - img.save(buf, format="PNG") - return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() - - -def _generate_backup_codes(count: int = 8) -> list: - """Generate single-use backup codes.""" - codes = [] - for _ in range(count): - # 8-digit numeric codes, hyphenated for readability - raw = secrets.randbelow(10_000_000_000) - code = f"{raw:010d}"[:8] - codes.append(f"{code[:4]}-{code[4:]}") - return codes - - -def _hash_backup_code(code: str) -> str: - """Hash a backup code for storage (same as password hashing).""" - return hash_password(code) - - -def _verify_backup_code(code: str, hashed: str) -> bool: - """Verify a backup code against its hash.""" - return verify_password(code, hashed) - - # ── Config ── # ── Storage (Redis-backed user store) ── -def _get_user(user_id: str) -> dict[str, Any] | None: - """Fetch user record by id from Redis hash rmi:users.""" - r = get_redis() - if not r: - return None - raw = r.hget("rmi:users", user_id) - if not raw: - return None - try: - return json.loads(raw) - except (json.JSONDecodeError, TypeError): - return None - - -def _get_user_by_email(email: str) -> dict[str, Any] | None: - """Fetch user record by email via rmi:users:email index → rmi:users hash.""" - r = get_redis() - if not r: - return None - user_id = r.hget("rmi:users:email", email.lower()) - if not user_id: - return None - if isinstance(user_id, bytes): - user_id = user_id.decode("utf-8") - return _get_user(user_id) - - -def _save_user(user: dict[str, Any]) -> None: - """Persist user record to Redis hash rmi:users, refresh email index if present.""" - r = get_redis() - if not r: - return - user_id = user["id"] - r.hset("rmi:users", user_id, json.dumps(user)) - if user.get("email"): - r.hset("rmi:users:email", user["email"].lower(), user_id) - - -# ── Helpers ── -def _is_valid_email(email: str) -> bool: - """Basic email validation.""" - pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$" - return re.match(pattern, email) is not None - - -# ── Storage (Redis-backed user store) ── -class EmailLoginRequest(BaseModel): - email: str - password: str - - -class EmailRegisterRequest(BaseModel): - email: str - password: str - display_name: str - wallet_address: str | None = None - wallet_chain: str | None = None - - -class WalletNonceRequest(BaseModel): - address: str - chain: str - - -class WalletVerifyRequest(BaseModel): - address: str - chain: str - nonce: str - signature: str - message: str - - -class NonceResponse(BaseModel): - nonce: str - timestamp: str - message: str | None = None - - -class WalletAuthResponse(BaseModel): - access_token: str - refresh_token: str - user: dict[str, Any] - - -class UserResponse(BaseModel): - id: str - email: str - display_name: str - wallet_address: str | None - wallet_chain: str | None - tier: str - role: str - created_at: str - xp: int = 0 - level: int = 1 - badges: list = [] - - -class GoogleAuthResponse(BaseModel): - url: str - - -class TelegramAuthRequest(BaseModel): - id: int - first_name: str | None = None - last_name: str | None = None - username: str | None = None - photo_url: str | None = None - auth_date: int - hash: str - - -# ── 2FA Models ── -class TwoFASetupResponse(BaseModel): - secret: str - qr_code: str # base64 data URI - uri: str # otpauth:// URI - backup_codes: list # plaintext - shown once - - -class TwoFAEnableRequest(BaseModel): - code: str = Field(..., min_length=6, max_length=6, pattern=r"^\d{6}$") - - -class TwoFAVerifyRequest(BaseModel): - code: str = Field(..., min_length=6, max_length=6, pattern=r"^\d{6}$") - - -class TwoFALoginRequest(BaseModel): - email: str - password: str - code: str = Field(..., min_length=6, max_length=10) # 6 digits or backup code XXXX-XXXX - - -class TwoFAStatusResponse(BaseModel): - enabled: bool - method: str = "totp" # future: u2f, webauthn - created_at: str | None = None - last_used: str | None = None - - # ── Email Auth ── @router.post("/register", response_model=WalletAuthResponse) def register_email(req: EmailRegisterRequest): @@ -416,7 +241,7 @@ async def wallet_verify(req: WalletVerifyRequest): # ── User Profile ── @router.get("/user/me", response_model=UserResponse) -async def get_current_user(request: Request): +async def user_me(request: Request): """Get current authenticated user profile.""" auth_header = request.headers.get("Authorization", "") if not auth_header.startswith("Bearer "): @@ -947,30 +772,6 @@ async def x_callback(request: Request): return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=x") -# ── FastAPI Dependency Functions (for @router/@app endpoints) ── -async def get_current_user(request: Request) -> dict[str, Any] | None: # noqa: F811 - """Verify JWT from Authorization header (wallet or email).""" - auth_header = request.headers.get("Authorization", "") - if not auth_header.startswith("Bearer "): - return None - token = auth_header[7:] - payload = _verify_jwt(token) - if not payload: - return None - user = _get_user(payload["id"]) - if not user: - return None - return user - - -async def require_auth(request: Request) -> dict[str, Any]: - """Require valid JWT. Raises 401 if missing/invalid.""" - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - return user - - # ── 2FA / TOTP Endpoints ── @@ -1095,30 +896,6 @@ async def twofa_status(request: Request): } -# ── Privacy Enforcement Dependency ── - - -async def require_public_profile(request: Request): - """ - Dependency to ensure the user has a public profile. - Required for actions that interact with the community (posting, commenting). - """ - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - - privacy_settings = user.get("privacy_settings", {}) - visibility = privacy_settings.get("profile_visibility", "public") - - if visibility != "public": - raise HTTPException( - status_code=403, - detail="A public profile is required to post or comment. Please update your privacy settings in your profile.", - ) - - return user - - @router.post("/2fa/verify") async def twofa_verify(req: TwoFAVerifyRequest, request: Request): """Verify a TOTP code (for re-auth during sensitive operations).""" diff --git a/app/domains/auth/deps.py b/app/domains/auth/deps.py index da98797..7d2f66f 100644 --- a/app/domains/auth/deps.py +++ b/app/domains/auth/deps.py @@ -1,12 +1,61 @@ -"""FastAPI dependencies - re-exports from app.domains.auth. +"""FastAPI dependencies for auth domain. Phase 3B of AUDIT-2026-Q3.md. Public API: -- get_current_user, require_auth, require_public_profile +- get_current_user +- require_auth +- require_public_profile """ -from app.auth import ( # noqa: F401 - get_current_user, - require_auth, - require_public_profile, -) +from __future__ import annotations + +from typing import Any + +from fastapi import HTTPException, Request + +from app.domains.auth.jwt import _verify_jwt +from app.domains.auth.store import _get_user + + +async def get_current_user(request: Request) -> dict[str, Any] | None: + """Verify JWT from Authorization header (wallet or email).""" + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + return None + token = auth_header[7:] + payload = _verify_jwt(token) + if not payload: + return None + user = _get_user(payload["id"]) + if not user: + return None + return user + + +async def require_auth(request: Request) -> dict[str, Any]: + """Require valid JWT. Raises 401 if missing/invalid.""" + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + return user + + +async def require_public_profile(request: Request) -> dict[str, Any]: + """ + Dependency to ensure the user has a public profile. + Required for actions that interact with the community (posting, commenting). + """ + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + + privacy_settings = user.get("privacy_settings", {}) + visibility = privacy_settings.get("profile_visibility", "public") + + if visibility != "public": + raise HTTPException( + status_code=403, + detail="A public profile is required to post or comment. Please update your privacy settings in your profile.", + ) + + return user diff --git a/app/domains/auth/schemas.py b/app/domains/auth/schemas.py index 0720322..5a964f1 100644 --- a/app/domains/auth/schemas.py +++ b/app/domains/auth/schemas.py @@ -1,27 +1,106 @@ -"""Pydantic schemas - re-exports from app.domains.auth. +"""Pydantic schemas for auth domain. Phase 3B of AUDIT-2026-Q3.md. - -Public API: -- EmailLoginRequest, EmailRegisterRequest -- WalletNonceRequest, WalletVerifyRequest, WalletAuthResponse -- NonceResponse, UserResponse, GoogleAuthResponse -- TelegramAuthRequest, TwoFAEnableRequest, TwoFALoginRequest, - TwoFASetupResponse, TwoFAStatusResponse, TwoFAVerifyRequest """ -from app.auth import ( # noqa: F401 - EmailLoginRequest, - EmailRegisterRequest, - GoogleAuthResponse, - NonceResponse, - TelegramAuthRequest, - TwoFAEnableRequest, - TwoFALoginRequest, - TwoFASetupResponse, - TwoFAStatusResponse, - TwoFAVerifyRequest, - UserResponse, - WalletAuthResponse, - WalletNonceRequest, - WalletVerifyRequest, -) +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class EmailLoginRequest(BaseModel): + email: str + password: str + + +class EmailRegisterRequest(BaseModel): + email: str + password: str + display_name: str + wallet_address: str | None = None + wallet_chain: str | None = None + + +class WalletNonceRequest(BaseModel): + address: str + chain: str + + +class WalletVerifyRequest(BaseModel): + address: str + chain: str + nonce: str + signature: str + message: str + + +class NonceResponse(BaseModel): + nonce: str + timestamp: str + message: str | None = None + + +class WalletAuthResponse(BaseModel): + access_token: str + refresh_token: str + user: dict[str, Any] + + +class UserResponse(BaseModel): + id: str + email: str + display_name: str + wallet_address: str | None + wallet_chain: str | None + tier: str + role: str + created_at: str + xp: int = 0 + level: int = 1 + badges: list = [] + + +class GoogleAuthResponse(BaseModel): + url: str + + +class TelegramAuthRequest(BaseModel): + id: int + first_name: str | None = None + last_name: str | None = None + username: str | None = None + photo_url: str | None = None + auth_date: int + hash: str + + +# ── 2FA Models ── +class TwoFASetupResponse(BaseModel): + secret: str + qr_code: str # base64 data URI + uri: str # otpauth:// URI + backup_codes: list # plaintext - shown once + + +class TwoFAEnableRequest(BaseModel): + code: str = Field(..., min_length=6, max_length=6, pattern=r"^\d{6}$") + + +class TwoFAVerifyRequest(BaseModel): + code: str = Field(..., min_length=6, max_length=6, pattern=r"^\d{6}$") + + +class TwoFALoginRequest(BaseModel): + email: str + password: str + code: str = Field(..., min_length=6, max_length=10) # 6 digits or backup code XXXX-XXXX + + +class TwoFAStatusResponse(BaseModel): + enabled: bool + method: str = "totp" # future: u2f, webauthn + created_at: str | None = None + last_used: str | None = None + + diff --git a/app/domains/auth/store.py b/app/domains/auth/store.py index eabd2bb..63be999 100644 --- a/app/domains/auth/store.py +++ b/app/domains/auth/store.py @@ -1,4 +1,4 @@ -"""User storage helpers - re-exports from app.domains.auth. +"""User storage helpers. Phase 3B of AUDIT-2026-Q3.md. @@ -6,10 +6,65 @@ Public API: - _get_user, _get_user_by_email, _save_user, _delete_user - _is_valid_email """ -from app.auth import ( # noqa: F401 - _delete_user, - _get_user, - _get_user_by_email, - _is_valid_email, - _save_user, -) +from __future__ import annotations + +import json +import re +from typing import Any + +from app.core.redis import get_redis + + +def _get_user(user_id: str) -> dict[str, Any] | None: + """Fetch user record by id from Redis hash rmi:users.""" + r = get_redis() + if not r: + return None + raw = r.hget("rmi:users", user_id) + if not raw: + return None + try: + return json.loads(raw) + except (json.JSONDecodeError, TypeError): + return None + + +def _get_user_by_email(email: str) -> dict[str, Any] | None: + """Fetch user record by email via rmi:users:email index -> rmi:users hash.""" + r = get_redis() + if not r: + return None + user_id = r.hget("rmi:users:email", email.lower()) + if not user_id: + return None + if isinstance(user_id, bytes): + user_id = user_id.decode("utf-8") + return _get_user(user_id) + + +def _save_user(user: dict[str, Any]) -> None: + """Persist user record to Redis hash rmi:users, refresh email index if present.""" + r = get_redis() + if not r: + return + user_id = user["id"] + r.hset("rmi:users", user_id, json.dumps(user)) + if user.get("email"): + r.hset("rmi:users:email", user["email"].lower(), user_id) + + +def _delete_user(user_id: str) -> None: + """Remove user record from Redis hash rmi:users and email index.""" + r = get_redis() + if not r: + return + user = _get_user(user_id) + if user and user.get("email"): + r.hdel("rmi:users:email", user["email"].lower()) + r.hdel("rmi:users", user_id) + + +def _is_valid_email(email: str) -> bool: + """Basic email validation.""" + pattern = r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$" + return re.match(pattern, email) is not None diff --git a/app/domains/auth/totp.py b/app/domains/auth/totp.py index 6898773..a84c95f 100644 --- a/app/domains/auth/totp.py +++ b/app/domains/auth/totp.py @@ -1,24 +1,92 @@ -"""TOTP / 2FA helpers - re-exports from app.domains.auth. +"""TOTP / 2FA helpers. Phase 3B of AUDIT-2026-Q3.md. Public API: -- _generate_totp_secret, _build_totp, _verify_totp, _get_totp_uri -- _generate_qr_base64, _generate_backup_codes, _hash_backup_code, _verify_backup_code -- TOTP_ISSUER, PYOTP_AVAILABLE +- PYOTP_AVAILABLE +- TOTP_ISSUER, TOTP_DIGITS, TOTP_INTERVAL, TOTP_WINDOW +- _generate_totp_secret, _build_totp, _verify_totp +- _get_totp_uri, _generate_qr_base64 +- _generate_backup_codes, _hash_backup_code, _verify_backup_code """ -from app.auth import ( # noqa: F401 - PYOTP_AVAILABLE, - TOTP_DIGITS, - TOTP_INTERVAL, - TOTP_ISSUER, - TOTP_WINDOW, - _build_totp, - _generate_backup_codes, - _generate_qr_base64, - _generate_totp_secret, - _get_totp_uri, - _hash_backup_code, - _verify_backup_code, - _verify_totp, -) +from __future__ import annotations + +import base64 +import io +import logging +import os +import secrets +from typing import Any + +from app.domains.auth.passwords import hash_password, verify_password + +logger = logging.getLogger(__name__) + +try: + import pyotp + import qrcode + + PYOTP_AVAILABLE = True +except ImportError: + PYOTP_AVAILABLE = False + logger.warning("pyotp not installed - 2FA endpoints will return 503") + +TOTP_ISSUER = os.getenv("TOTP_ISSUER", "RugMunch Intelligence") +TOTP_DIGITS = 6 +TOTP_INTERVAL = 30 +TOTP_WINDOW = 1 # Allow ±1 interval clock drift + + +def _generate_totp_secret() -> str: + """Generate a new base32 TOTP secret.""" + if not PYOTP_AVAILABLE: + raise RuntimeError("pyotp not installed") + return pyotp.random_base32() + + +def _build_totp(secret: str) -> Any: + """Build a TOTP object from secret.""" + return pyotp.TOTP(secret, digits=TOTP_DIGITS, interval=TOTP_INTERVAL) + + +def _verify_totp(secret: str, code: str) -> bool: + """Verify a TOTP code with clock-drift tolerance.""" + if not PYOTP_AVAILABLE or not secret or not code: + return False + totp = _build_totp(secret) + return totp.verify(code, valid_window=TOTP_WINDOW) + + +def _get_totp_uri(secret: str, email: str) -> str: + """Get otpauth:// URI for QR code generation.""" + totp = _build_totp(secret) + return totp.provisioning_uri(name=email, issuer_name=TOTP_ISSUER) + + +def _generate_qr_base64(uri: str) -> str: + """Generate QR code as base64 PNG data URI.""" + img = qrcode.make(uri) + buf = io.BytesIO() + img.save(buf, format="PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + + +def _generate_backup_codes(count: int = 8) -> list: + """Generate single-use backup codes.""" + codes = [] + for _ in range(count): + # 8-digit numeric codes, hyphenated for readability + raw = secrets.randbelow(10_000_000_000) + code = f"{raw:010d}"[:8] + codes.append(f"{code[:4]}-{code[4:]}") + return codes + + +def _hash_backup_code(code: str) -> str: + """Hash a backup code for storage (same as password hashing).""" + return hash_password(code) + + +def _verify_backup_code(code: str, hashed: str) -> bool: + """Verify a backup code against its hash.""" + return verify_password(code, hashed) From f5e1e140dc75905ec8a2a12b4ea896e1c50a048f Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 14:50:16 +0700 Subject: [PATCH 06/12] refactor(auth): split OAuth + Telegram endpoints to app.domains.auth.oauth (P4.3 cont) --- app/domains/auth/__init__.py | 510 +---------------------------------- app/domains/auth/oauth.py | 424 +++++++++++++++++++++++++++-- 2 files changed, 414 insertions(+), 520 deletions(-) diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index 25cb745..4479e4d 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -4,7 +4,6 @@ Auth Router - Complete authentication system (email, wallet, OAuth, Telegram) import json import logging -import os from datetime import datetime, timedelta from fastapi import APIRouter, HTTPException, Request @@ -24,13 +23,14 @@ from app.domains.auth.jwt import ( _verify_jwt, generate_nonce, ) +from app.domains.auth.oauth import router as oauth_router from app.domains.auth.passwords import hash_password, verify_password from app.domains.auth.schemas import ( EmailLoginRequest, EmailRegisterRequest, - GoogleAuthResponse, + GoogleAuthResponse as GoogleAuthResponse, NonceResponse, - TelegramAuthRequest, + TelegramAuthRequest as TelegramAuthRequest, TwoFAEnableRequest, TwoFALoginRequest, TwoFASetupResponse as TwoFASetupResponse, @@ -68,6 +68,9 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["auth"]) +# Include OAuth/Telegram routes from dedicated submodule +router.include_router(oauth_router, tags=["auth"]) + # ── Config ── @@ -271,507 +274,6 @@ async def user_me(request: Request): } -# ── Google OAuth ── -@router.get("/google/url", response_model=GoogleAuthResponse) -async def google_auth_url(): - """Get Google OAuth URL.""" - client_id = os.getenv("GOOGLE_CLIENT_ID") - redirect_uri = os.getenv( - "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" - ) - - if not client_id: - return {"url": "/auth/callback?error=google_not_configured"} - - from urllib.parse import urlencode - - params = { - "client_id": client_id, - "redirect_uri": redirect_uri, - "response_type": "code", - "scope": "openid email profile", - "access_type": "offline", - "prompt": "consent", - } - url = f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" - return {"url": url} - - -FRONTEND_URL = os.getenv("FRONTEND_URL", "https://rugmunch.io") - - -@router.get("/google/callback") -async def google_callback(request: Request): - """Handle Google OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - - code = request.query_params.get("code") - error = request.query_params.get("error") - - if error: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error={error}") - - if not code: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=missing_code") - - import httpx - - client_id = os.getenv("GOOGLE_CLIENT_ID") - client_secret = os.getenv("GOOGLE_CLIENT_SECRET") - redirect_uri = os.getenv( - "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" - ) - - if not client_id or not client_secret: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") - - async with httpx.AsyncClient() as client: - resp = await client.post( - "https://oauth2.googleapis.com/token", - data={ - "code": code, - "client_id": client_id, - "client_secret": client_secret, - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - }, - ) - if resp.status_code != 200: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=token_exchange_failed") - - tokens = resp.json() - access_token = tokens.get("access_token") - - resp = await client.get( - "https://www.googleapis.com/oauth2/v2/userinfo", - headers={"Authorization": f"Bearer {access_token}"}, - ) - if resp.status_code != 200: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=userinfo_failed") - - google_user = resp.json() - email = google_user.get("email") - - user = _get_user_by_email(email) - if not user: - user_id = _derive_user_id(email) - user = { - "id": user_id, - "email": email, - "display_name": google_user.get("name", email), - "tier": "FREE", - "role": "USER", - "created_at": datetime.utcnow().isoformat(), - "xp": 0, - "level": 1, - "badges": [], - "scans_remaining": 5, - "scans_used": 0, - } - r = get_redis() - r.hset("rmi:users", user_id, json.dumps(user)) - r.hset("rmi:users:email", email.lower(), user_id) - - jwt_token = _create_jwt( - user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") - ) - - # Redirect to frontend with token - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=google") - - -@router.post("/google/callback", response_model=WalletAuthResponse) -async def google_callback_post(request: Request): - """Legacy POST endpoint for manual code exchange.""" - body = await request.json() - code = body.get("code") - - if not code: - raise HTTPException(status_code=400, detail="Missing authorization code") - - import httpx - - client_id = os.getenv("GOOGLE_CLIENT_ID") - client_secret = os.getenv("GOOGLE_CLIENT_SECRET") - redirect_uri = os.getenv( - "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" - ) - - if not client_id or not client_secret: - raise HTTPException(status_code=500, detail="Google OAuth not configured") - - async with httpx.AsyncClient() as client: - resp = await client.post( - "https://oauth2.googleapis.com/token", - data={ - "code": code, - "client_id": client_id, - "client_secret": client_secret, - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - }, - ) - if resp.status_code != 200: - raise HTTPException(status_code=400, detail="Failed to exchange code") - - tokens = resp.json() - access_token = tokens.get("access_token") - - resp = await client.get( - "https://www.googleapis.com/oauth2/v2/userinfo", - headers={"Authorization": f"Bearer {access_token}"}, - ) - if resp.status_code != 200: - raise HTTPException(status_code=400, detail="Failed to get user info") - - google_user = resp.json() - email = google_user.get("email") - - user = _get_user_by_email(email) - if not user: - user_id = _derive_user_id(email) - user = { - "id": user_id, - "email": email, - "display_name": google_user.get("name", email), - "tier": "FREE", - "role": "USER", - "created_at": datetime.utcnow().isoformat(), - "xp": 0, - "level": 1, - "badges": [], - "scans_remaining": 5, - "scans_used": 0, - } - r = get_redis() - r.hset("rmi:users", user_id, json.dumps(user)) - r.hset("rmi:users:email", email.lower(), user_id) - - jwt_token = _create_jwt( - user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") - ) - - return { - "access_token": jwt_token, - "refresh_token": jwt_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", email), - "wallet_address": None, - "wallet_chain": None, - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - }, - } - - -# ── Telegram Auth ── -@router.post("/telegram", response_model=WalletAuthResponse) -async def telegram_auth(req: TelegramAuthRequest): - """Authenticate via Telegram Web App data.""" - bot_token = os.getenv("TELEGRAM_BOT_TOKEN") - if not bot_token: - raise HTTPException(status_code=500, detail="Telegram auth not configured") - - data_check = { - "id": str(req.id), - "auth_date": str(req.auth_date), - } - if req.first_name: - data_check["first_name"] = req.first_name - if req.last_name: - data_check["last_name"] = req.last_name - if req.username: - data_check["username"] = req.username - if req.photo_url: - data_check["photo_url"] = req.photo_url - - import hashlib - import hmac - - sorted_data = "\n".join(f"{k}={v}" for k, v in sorted(data_check.items())) - secret = hashlib.sha256(bot_token.encode()).digest() - expected_hash = hmac.new(secret, sorted_data.encode(), hashlib.sha256).hexdigest() - - if not hmac.compare_digest(req.hash, expected_hash): - raise HTTPException(status_code=401, detail="Invalid Telegram auth data") - - telegram_user_id = f"tg:{req.id}" - user = _get_user(telegram_user_id) - - if not user: - display_name = ( - f"{req.first_name or ''} {req.last_name or ''}".strip() - or req.username - or f"User{req.id}" - ) - email = f"{req.id}@telegram.rmi" - - user = { - "id": telegram_user_id, - "email": email, - "display_name": display_name, - "telegram_id": req.id, - "telegram_username": req.username, - "tier": "FREE", - "role": "USER", - "created_at": datetime.utcnow().isoformat(), - "xp": 0, - "level": 1, - "badges": [], - "scans_remaining": 5, - "scans_used": 0, - } - r = get_redis() - r.hset("rmi:users", telegram_user_id, json.dumps(user)) - r.hset("rmi:users:telegram", str(req.id), telegram_user_id) - - jwt_token = _create_jwt( - user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") - ) - - return { - "access_token": jwt_token, - "refresh_token": jwt_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name"), - "wallet_address": None, - "wallet_chain": None, - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - }, - } - - -# ── GitHub OAuth ── -@router.get("/github/url", response_model=GoogleAuthResponse) -async def github_auth_url(): - """Get GitHub OAuth URL.""" - client_id = os.getenv("GITHUB_CLIENT_ID") - redirect_uri = os.getenv( - "GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback" - ) - - if not client_id: - return {"url": "/auth/callback?error=github_not_configured"} - - from urllib.parse import urlencode - - params = { - "client_id": client_id, - "redirect_uri": redirect_uri, - "scope": "user:email", - } - url = f"https://github.com/login/oauth/authorize?{urlencode(params)}" - return {"url": url} - - -@router.get("/github/callback") -async def github_callback(request: Request): - """Handle GitHub OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - - code = request.query_params.get("code") - error = request.query_params.get("error") - - if error: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error={error}") - - if not code: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=missing_code") - - import httpx - - client_id = os.getenv("GITHUB_CLIENT_ID") - client_secret = os.getenv("GITHUB_CLIENT_SECRET") - redirect_uri = os.getenv( - "GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback" - ) - - if not client_id or not client_secret: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") - - async with httpx.AsyncClient() as client: - # Exchange code for access token - resp = await client.post( - "https://github.com/login/oauth/access_token", - data={ - "client_id": client_id, - "client_secret": client_secret, - "code": code, - "redirect_uri": redirect_uri, - }, - headers={"Accept": "application/json"}, - ) - if resp.status_code != 200: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=token_exchange_failed") - - tokens = resp.json() - access_token = tokens.get("access_token") - - # Get user info - resp = await client.get( - "https://api.github.com/user", - headers={"Authorization": f"token {access_token}", "Accept": "application/json"}, - ) - if resp.status_code != 200: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=userinfo_failed") - - github_user = resp.json() - email = github_user.get("email") - - # If no public email, fetch emails endpoint - if not email: - resp = await client.get( - "https://api.github.com/user/emails", - headers={"Authorization": f"token {access_token}", "Accept": "application/json"}, - ) - if resp.status_code == 200: - emails = resp.json() - primary = next((e for e in emails if e.get("primary")), None) - email = ( - primary.get("email") - if primary - else (emails[0].get("email") if emails else None) - ) - - if not email: - email = f"{github_user.get('login', 'github_user')}@github.rmi" - - user = _get_user_by_email(email) - if not user: - user_id = _derive_user_id(email) - user = { - "id": user_id, - "email": email, - "display_name": github_user.get("name", github_user.get("login", email)), - "tier": "FREE", - "role": "USER", - "created_at": datetime.utcnow().isoformat(), - "xp": 0, - "level": 1, - "badges": [], - "scans_remaining": 5, - "scans_used": 0, - } - r = get_redis() - r.hset("rmi:users", user_id, json.dumps(user)) - r.hset("rmi:users:email", email.lower(), user_id) - - jwt_token = _create_jwt( - user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") - ) - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=github") - - -# ── X/Twitter OAuth ── -@router.get("/x/url", response_model=GoogleAuthResponse) -async def x_auth_url(): - """Get X/Twitter OAuth URL.""" - client_id = os.getenv("X_CLIENT_ID") - redirect_uri = os.getenv("X_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/x/callback") - - if not client_id: - return {"url": "/auth/callback?error=x_not_configured"} - - from urllib.parse import urlencode - - params = { - "client_id": client_id, - "redirect_uri": redirect_uri, - "response_type": "code", - "scope": "tweet.read users.read", - } - url = f"https://twitter.com/i/oauth2/authorize?{urlencode(params)}" - return {"url": url} - - -@router.get("/x/callback") -async def x_callback(request: Request): - """Handle X/Twitter OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - - code = request.query_params.get("code") - error = request.query_params.get("error") - - if error: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error={error}") - - if not code: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=missing_code") - - import httpx - - client_id = os.getenv("X_CLIENT_ID") - client_secret = os.getenv("X_CLIENT_SECRET") - redirect_uri = os.getenv("X_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/x/callback") - - if not client_id or not client_secret: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") - - async with httpx.AsyncClient() as client: - # X uses OAuth 2.0 - exchange code for token - resp = await client.post( - "https://api.twitter.com/2/oauth2/token", - data={ - "code": code, - "grant_type": "authorization_code", - "client_id": client_id, - "redirect_uri": redirect_uri, - "code_verifier": "challenge", # X requires PKCE - we need to implement proper PKCE - }, - auth=(client_id, client_secret), - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - if resp.status_code != 200: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=token_exchange_failed") - - tokens = resp.json() - access_token = tokens.get("access_token") - - # Get user info from X API - resp = await client.get( - "https://api.twitter.com/2/users/me", - headers={"Authorization": f"Bearer {access_token}"}, - ) - if resp.status_code != 200: - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=userinfo_failed") - - x_user = resp.json().get("data", {}) - username = x_user.get("username", f"x_user_{x_user.get('id', 'unknown')}") - email = f"{username}@x.rmi" # X API v2 doesn't always return email - - user = _get_user_by_email(email) - if not user: - user_id = _derive_user_id(email) - user = { - "id": user_id, - "email": email, - "display_name": x_user.get("name", username), - "tier": "FREE", - "role": "USER", - "created_at": datetime.utcnow().isoformat(), - "xp": 0, - "level": 1, - "badges": [], - "scans_remaining": 5, - "scans_used": 0, - } - r = get_redis() - r.hset("rmi:users", user_id, json.dumps(user)) - r.hset("rmi:users:email", email.lower(), user_id) - - jwt_token = _create_jwt( - user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") - ) - return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=x") - - # ── 2FA / TOTP Endpoints ── diff --git a/app/domains/auth/oauth.py b/app/domains/auth/oauth.py index 8c63a8f..c2f4c27 100644 --- a/app/domains/auth/oauth.py +++ b/app/domains/auth/oauth.py @@ -1,20 +1,412 @@ -"""OAuth flows - re-exports from app.domains.auth. +"""OAuth + Telegram auth endpoints. Phase 3B of AUDIT-2026-Q3.md. - -Public API: -- google_auth_url, google_callback, google_callback_post -- github_auth_url, github_callback -- x_auth_url, x_callback -- telegram_auth """ -from app.auth import ( # noqa: F401 - github_auth_url, - github_callback, - google_auth_url, - google_callback, - google_callback_post, - telegram_auth, - x_auth_url, - x_callback, +from __future__ import annotations + +import json +import os +from datetime import datetime + +from fastapi import APIRouter, HTTPException, Request + +from app.core.redis import get_redis +from app.domains.auth.jwt import _create_jwt, _derive_user_id +from app.domains.auth.schemas import ( + GoogleAuthResponse, + TelegramAuthRequest, + WalletAuthResponse, ) +from app.domains.auth.store import _get_user, _get_user_by_email + +router = APIRouter(tags=["auth"]) + +FRONTEND_URL = os.getenv("FRONTEND_URL", "https://rugmunch.io") + + +@router.get("/google/url", response_model=GoogleAuthResponse) +async def google_auth_url(): + """Get Google OAuth URL.""" + client_id = os.getenv("GOOGLE_CLIENT_ID") + redirect_uri = os.getenv( + "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" + ) + + if not client_id: + return {"url": "/auth/callback?error=google_not_configured"} + + from urllib.parse import urlencode + + params = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": "openid email profile", + "access_type": "offline", + "prompt": "consent", + } + url = f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" + return {"url": url} + + +@router.get("/google/callback") +async def google_callback(request: Request): + """Handle Google OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" + from fastapi.responses import RedirectResponse + + code = request.query_params.get("code") + error = request.query_params.get("error") + + if error: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error={error}") + + if not code: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=missing_code") + + import httpx + + client_id = os.getenv("GOOGLE_CLIENT_ID") + client_secret = os.getenv("GOOGLE_CLIENT_SECRET") + redirect_uri = os.getenv( + "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" + ) + + if not client_id or not client_secret: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") + + async with httpx.AsyncClient() as client: + resp = await client.post( + "https://oauth2.googleapis.com/token", + data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + }, + ) + if resp.status_code != 200: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=token_exchange_failed") + + tokens = resp.json() + access_token = tokens.get("access_token") + + resp = await client.get( + "https://www.googleapis.com/oauth2/v2/userinfo", + headers={"Authorization": f"Bearer {access_token}"}, + ) + if resp.status_code != 200: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=userinfo_failed") + + google_user = resp.json() + email = google_user.get("email") + + user = _get_user_by_email(email) + if not user: + user_id = _derive_user_id(email) + user = { + "id": user_id, + "email": email, + "display_name": google_user.get("name", email), + "tier": "FREE", + "role": "USER", + "created_at": datetime.utcnow().isoformat(), + "xp": 0, + "level": 1, + "badges": [], + "scans_remaining": 5, + "scans_used": 0, + } + r = get_redis() + r.hset("rmi:users", user_id, json.dumps(user)) + r.hset("rmi:users:email", email.lower(), user_id) + + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) + + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=google") + + +@router.post("/google/callback", response_model=WalletAuthResponse) +async def google_callback_post(request: Request): + """Legacy POST endpoint for manual code exchange.""" + body = await request.json() + code = body.get("code") + + if not code: + raise HTTPException(status_code=400, detail="Missing authorization code") + + import httpx + + client_id = os.getenv("GOOGLE_CLIENT_ID") + client_secret = os.getenv("GOOGLE_CLIENT_SECRET") + redirect_uri = os.getenv( + "GOOGLE_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/google/callback" + ) + + if not client_id or not client_secret: + raise HTTPException(status_code=500, detail="Google OAuth not configured") + + async with httpx.AsyncClient() as client: + resp = await client.post( + "https://oauth2.googleapis.com/token", + data={ + "code": code, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + }, + ) + if resp.status_code != 200: + raise HTTPException(status_code=400, detail="Failed to exchange code") + + tokens = resp.json() + access_token = tokens.get("access_token") + + resp = await client.get( + "https://www.googleapis.com/oauth2/v2/userinfo", + headers={"Authorization": f"Bearer {access_token}"}, + ) + if resp.status_code != 200: + raise HTTPException(status_code=400, detail="Failed to get user info") + + google_user = resp.json() + email = google_user.get("email") + + user = _get_user_by_email(email) + if not user: + user_id = _derive_user_id(email) + user = { + "id": user_id, + "email": email, + "display_name": google_user.get("name", email), + "tier": "FREE", + "role": "USER", + "created_at": datetime.utcnow().isoformat(), + "xp": 0, + "level": 1, + "badges": [], + "scans_remaining": 5, + "scans_used": 0, + } + r = get_redis() + r.hset("rmi:users", user_id, json.dumps(user)) + r.hset("rmi:users:email", email.lower(), user_id) + + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) + + return { + "access_token": jwt_token, + "refresh_token": jwt_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", email), + "wallet_address": None, + "wallet_chain": None, + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, + } + + +@router.post("/telegram", response_model=WalletAuthResponse) +async def telegram_auth(req: TelegramAuthRequest): + """Authenticate via Telegram Web App data.""" + bot_token = os.getenv("TELEGRAM_BOT_TOKEN") + if not bot_token: + raise HTTPException(status_code=500, detail="Telegram auth not configured") + + data_check = { + "id": str(req.id), + "auth_date": str(req.auth_date), + } + if req.first_name: + data_check["first_name"] = req.first_name + if req.last_name: + data_check["last_name"] = req.last_name + if req.username: + data_check["username"] = req.username + if req.photo_url: + data_check["photo_url"] = req.photo_url + + import hashlib + import hmac + + sorted_data = "\n".join(f"{k}={v}" for k, v in sorted(data_check.items())) + secret = hashlib.sha256(bot_token.encode()).digest() + expected_hash = hmac.new(secret, sorted_data.encode(), hashlib.sha256).hexdigest() + + if not hmac.compare_digest(req.hash, expected_hash): + raise HTTPException(status_code=401, detail="Invalid Telegram auth data") + + telegram_user_id = f"tg:{req.id}" + user = _get_user(telegram_user_id) + + if not user: + display_name = ( + f"{req.first_name or ''} {req.last_name or ''}".strip() + or req.username + or f"User{req.id}" + ) + email = f"{req.id}@telegram.rmi" + + user = { + "id": telegram_user_id, + "email": email, + "display_name": display_name, + "telegram_id": req.id, + "telegram_username": req.username, + "tier": "FREE", + "role": "USER", + "created_at": datetime.utcnow().isoformat(), + "xp": 0, + "level": 1, + "badges": [], + "scans_remaining": 5, + "scans_used": 0, + } + r = get_redis() + r.hset("rmi:users", telegram_user_id, json.dumps(user)) + r.hset("rmi:users:telegram", str(req.id), telegram_user_id) + + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) + + return { + "access_token": jwt_token, + "refresh_token": jwt_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name"), + "wallet_address": None, + "wallet_chain": None, + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, + } + + +@router.get("/github/url", response_model=GoogleAuthResponse) +async def github_auth_url(): + """Get GitHub OAuth URL.""" + client_id = os.getenv("GITHUB_CLIENT_ID") + redirect_uri = os.getenv( + "GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback" + ) + + if not client_id: + return {"url": "/auth/callback?error=github_not_configured"} + + from urllib.parse import urlencode + + params = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "scope": "user:email", + } + url = f"https://github.com/login/oauth/authorize?{urlencode(params)}" + return {"url": url} + + +@router.get("/github/callback") +async def github_callback(request: Request): + """Handle GitHub OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" + from fastapi.responses import RedirectResponse + + code = request.query_params.get("code") + error = request.query_params.get("error") + + if error: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error={error}") + + if not code: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=missing_code") + + import httpx + + client_id = os.getenv("GITHUB_CLIENT_ID") + client_secret = os.getenv("GITHUB_CLIENT_SECRET") + redirect_uri = os.getenv( + "GITHUB_REDIRECT_URI", "https://rugmunch.io/api/v1/auth/github/callback" + ) + + if not client_id or not client_secret: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=not_configured") + + async with httpx.AsyncClient() as client: + resp = await client.post( + "https://github.com/login/oauth/access_token", + data={ + "client_id": client_id, + "client_secret": client_secret, + "code": code, + "redirect_uri": redirect_uri, + }, + headers={"Accept": "application/json"}, + ) + if resp.status_code != 200: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=token_exchange_failed") + + tokens = resp.json() + access_token = tokens.get("access_token") + + resp = await client.get( + "https://api.github.com/user", + headers={"Authorization": f"token {access_token}", "Accept": "application/json"}, + ) + if resp.status_code != 200: + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?error=userinfo_failed") + + github_user = resp.json() + email = github_user.get("email") + + if not email: + resp = await client.get( + "https://api.github.com/user/emails", + headers={"Authorization": f"token {access_token}", "Accept": "application/json"}, + ) + if resp.status_code == 200: + emails = resp.json() + primary = next((e for e in emails if e.get("primary")), None) + email = ( + primary.get("email") + if primary + else (emails[0].get("email") if emails else None) + ) + + if not email: + email = f"{github_user.get('login', 'github_user')}@github.rmi" + + user = _get_user_by_email(email) + if not user: + user_id = _derive_user_id(email) + user = { + "id": user_id, + "email": email, + "display_name": github_user.get("name", github_user.get("login", email)), + "tier": "FREE", + "role": "USER", + "created_at": datetime.utcnow().isoformat(), + "xp": 0, + "level": 1, + "badges": [], + "scans_remaining": 5, + "scans_used": 0, + } + r = get_redis() + r.hset("rmi:users", user_id, json.dumps(user)) + r.hset("rmi:users:email", email.lower(), user_id) + + jwt_token = _create_jwt( + user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") + ) + return RedirectResponse(f"{FRONTEND_URL}/auth/callback?token={jwt_token}&provider=github") From 018edaded79b2dbb200e576ba2a2fa7373ee7fe6 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 14:53:21 +0700 Subject: [PATCH 07/12] refactor(auth): archive app/auth.py + app/auth_wallet.py, migrate all imports to app.domains.auth.* (P4.3 cont) --- app/{ => _archive/legacy_2026_07}/auth.py | 0 .../legacy_2026_07}/auth_wallet.py | 0 app/admin_backend.py | 4 +-- app/domains/billing/x402/tools/token_tools.py | 4 +-- app/routers/admin_backend.py | 2 +- app/routers/admin_users_api.py | 22 ++++++------- app/routers/agents.py | 2 +- app/routers/alerts.py | 3 +- app/routers/auth_extensions.py | 31 ++++++++++++------- app/routers/bulletin.py | 2 +- app/routers/cases_investigators_api.py | 12 +++---- app/routers/chat.py | 2 +- app/routers/news.py | 2 +- app/routers/scam_school.py | 2 +- app/routers/subscription_pricing_api.py | 18 +++++------ app/routers/x402_dashboard.py | 2 +- app/routers/x402_middleware.py | 12 +++---- app/routers/x402_settle_api.py | 4 +-- app/scan_rate_limiter.py | 2 +- 19 files changed, 68 insertions(+), 58 deletions(-) rename app/{ => _archive/legacy_2026_07}/auth.py (100%) rename app/{ => _archive/legacy_2026_07}/auth_wallet.py (100%) diff --git a/app/auth.py b/app/_archive/legacy_2026_07/auth.py similarity index 100% rename from app/auth.py rename to app/_archive/legacy_2026_07/auth.py diff --git a/app/auth_wallet.py b/app/_archive/legacy_2026_07/auth_wallet.py similarity index 100% rename from app/auth_wallet.py rename to app/_archive/legacy_2026_07/auth_wallet.py diff --git a/app/admin_backend.py b/app/admin_backend.py index 7faa91f..dfb357e 100644 --- a/app/admin_backend.py +++ b/app/admin_backend.py @@ -819,7 +819,7 @@ class AdminUserStore: created_by: str = "", ) -> dict | None: """Create a new admin user.""" - from app.auth import hash_password + from app.domains.auth.passwords import hash_password # Check if email already exists existing = await AdminUserStore.get_admin_by_email(email) @@ -853,7 +853,7 @@ class AdminUserStore: @staticmethod async def verify_admin_login(email: str, password: str) -> dict | None: """Verify admin login credentials.""" - from app.auth import verify_password + from app.domains.auth.passwords import verify_password admin = await AdminUserStore.get_admin_by_email(email) if not admin: diff --git a/app/domains/billing/x402/tools/token_tools.py b/app/domains/billing/x402/tools/token_tools.py index 7b149e5..7555b07 100644 --- a/app/domains/billing/x402/tools/token_tools.py +++ b/app/domains/billing/x402/tools/token_tools.py @@ -51,7 +51,7 @@ async def deep_contract_audit(req: TokenRequest): # If primary returned no data, use full fallback chain if not result.get("sources_used"): - from app.auth import get_redis as _get_redis + from app.core.redis import get_redis as _get_redis from app.fallback_engine import try_all_fallbacks r = await _get_redis() @@ -72,7 +72,7 @@ async def deep_contract_audit(req: TokenRequest): except Exception as e: # Last resort: try fallback before raising error try: - from app.auth import get_redis as _get_redis + from app.core.redis import get_redis as _get_redis from app.fallback_engine import try_all_fallbacks r = await _get_redis() diff --git a/app/routers/admin_backend.py b/app/routers/admin_backend.py index b55620d..e5b662d 100644 --- a/app/routers/admin_backend.py +++ b/app/routers/admin_backend.py @@ -1128,7 +1128,7 @@ async def reset_admin_password(request: Request, admin_id: str, body: dict = Bod updater = auth["admin"] ip, ua = _get_client_info(request) - from app.auth import hash_password + from app.domains.auth.passwords import hash_password admin = await AdminUserStore.get_admin(admin_id) if not admin: diff --git a/app/routers/admin_users_api.py b/app/routers/admin_users_api.py index 215b305..dfb278d 100644 --- a/app/routers/admin_users_api.py +++ b/app/routers/admin_users_api.py @@ -33,7 +33,7 @@ router = APIRouter(tags=["admin-users"]) # ── Redis helper ── async def require_admin(request: Request, min_role: str = "ADMIN"): """Require admin authentication.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -131,7 +131,7 @@ def _get_all_users() -> list[dict]: def _get_user_full(user_id: str) -> dict | None: """Get user with enriched data.""" - from app.auth import _get_user + from app.domains.auth.store import _get_user user = _get_user(user_id) if not user: @@ -285,7 +285,7 @@ async def warn_user(user_id: str, req: WarnUserRequest, request: Request): """Add a warning to a user.""" admin = await require_admin(request, min_role="MODERATOR") - from app.auth import _get_user + from app.domains.auth.store import _get_user user = _get_user(user_id) if not user: @@ -320,7 +320,7 @@ async def flag_suspect(user_id: str, request: Request, reason: str | None = Quer """Flag a user as suspicious.""" admin = await require_admin(request, min_role="MODERATOR") - from app.auth import _get_user + from app.domains.auth.store import _get_user user = _get_user(user_id) if not user: @@ -360,7 +360,7 @@ async def ban_user(user_id: str, request: Request, reason: str | None = Query(No """Ban a user.""" admin = await require_admin(request, min_role="ADMIN") - from app.auth import _get_user + from app.domains.auth.store import _get_user user = _get_user(user_id) if not user: @@ -402,7 +402,7 @@ async def delete_user(user_id: str, request: Request, reason: str | None = Query """Soft delete a user.""" admin = await require_admin(request, min_role="ADMIN") - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user user = _get_user(user_id) if not user: @@ -430,7 +430,7 @@ async def restore_user(user_id: str, request: Request): """Restore a soft-deleted user.""" admin = await require_admin(request, min_role="ADMIN") - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user user = _get_user(user_id) if not user: @@ -454,7 +454,7 @@ async def update_user(user_id: str, req: UpdateUserRequest, request: Request): """Update user properties (tier, role, status, etc.).""" admin = await require_admin(request, min_role="ADMIN") - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user user = _get_user(user_id) if not user: @@ -530,7 +530,7 @@ async def bulk_action(req: BulkActionRequest, request: Request): r.hset("rmi:user_status", user_id, "active") results["success"].append({"id": user_id, "action": "cleared_suspect"}) elif req.action == "delete": - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user user = _get_user(user_id) if user: @@ -541,7 +541,7 @@ async def bulk_action(req: BulkActionRequest, request: Request): r.hset("rmi:user_status", user_id, "deleted") results["success"].append({"id": user_id, "action": "deleted"}) elif req.action == "restore": - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user user = _get_user(user_id) if user: @@ -553,7 +553,7 @@ async def bulk_action(req: BulkActionRequest, request: Request): results["success"].append({"id": user_id, "action": "restored"}) elif req.action == "update_tier": tier = req.params.get("tier", "FREE") if req.params else "FREE" - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user user = _get_user(user_id) if user: diff --git a/app/routers/agents.py b/app/routers/agents.py index 3b0433e..19b068a 100644 --- a/app/routers/agents.py +++ b/app/routers/agents.py @@ -84,7 +84,7 @@ class AgentCommandRequest(BaseModel): priority: str = Field(default="normal") -from app.auth import get_redis # noqa: E402 +from app.domains.auth import get_redis # noqa: E402 # ── Static routes must comes before parameterized routes to avoid FastAPI matching issues ── diff --git a/app/routers/alerts.py b/app/routers/alerts.py index 4270293..0d0551c 100644 --- a/app/routers/alerts.py +++ b/app/routers/alerts.py @@ -21,7 +21,8 @@ class AlertRequest(BaseModel): webhook_url: str | None = None -from app.auth import get_redis, require_auth # noqa: E402 +from app.core.redis import get_redis +from app.domains.auth import require_auth # noqa: E402 def _get_redis_sync(): diff --git a/app/routers/auth_extensions.py b/app/routers/auth_extensions.py index d738922..efac72a 100644 --- a/app/routers/auth_extensions.py +++ b/app/routers/auth_extensions.py @@ -91,7 +91,7 @@ class ProfileResponse(BaseModel): @router.post("/forgot-password") async def forgot_password(req: ForgotPasswordRequest): """Request password reset - sends email with reset token.""" - from app.auth import _get_user_by_email + from app.domains.auth.store import _get_user_by_email user = _get_user_by_email(req.email) if not user: @@ -140,7 +140,8 @@ async def forgot_password(req: ForgotPasswordRequest): @router.post("/reset-password") async def reset_password(req: ResetPasswordRequest): """Reset password using token from email.""" - from app.auth import _get_user, _save_user, hash_password + from app.domains.auth.passwords import hash_password + from app.domains.auth.store import _get_user, _save_user token_hash = hashlib.sha256(req.token.encode()).hexdigest() r = get_redis() # noqa: F821 -- pre-existing bug, see fix(f821) tracking issue @@ -173,7 +174,9 @@ async def reset_password(req: ResetPasswordRequest): @router.post("/change-password") async def change_password(req: ChangePasswordRequest, request: Request): """Change password while logged in.""" - from app.auth import _save_user, get_current_user, hash_password, verify_password + from app.domains.auth import get_current_user + from app.domains.auth.passwords import hash_password, verify_password + from app.domains.auth.store import _save_user user = await get_current_user(request) if not user: @@ -200,7 +203,7 @@ async def change_password(req: ChangePasswordRequest, request: Request): @router.get("/profile", response_model=ProfileResponse) async def get_profile(request: Request): """Get full user profile including linked wallets.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -229,7 +232,8 @@ async def get_profile(request: Request): @router.patch("/profile") async def update_profile(req: UpdateProfileRequest, request: Request): """Update user profile (display name, avatar).""" - from app.auth import _save_user, get_current_user + from app.domains.auth import get_current_user + from app.domains.auth.store import _save_user user = await get_current_user(request) if not user: @@ -388,7 +392,8 @@ async def list_supported_chains(): @router.post("/wallet/link") async def link_wallet(req: LinkWalletRequest, request: Request): """Link a new wallet to the user's account.""" - from app.auth import _save_user, get_current_user + from app.domains.auth import get_current_user + from app.domains.auth.store import _save_user user = await get_current_user(request) if not user: @@ -437,7 +442,8 @@ async def link_wallet(req: LinkWalletRequest, request: Request): @router.post("/wallet/unlink") async def unlink_wallet(req: UnlinkWalletRequest, request: Request): """Unlink a wallet from the user's account.""" - from app.auth import _save_user, get_current_user + from app.domains.auth import get_current_user + from app.domains.auth.store import _save_user user = await get_current_user(request) if not user: @@ -476,7 +482,7 @@ async def unlink_wallet(req: UnlinkWalletRequest, request: Request): @router.get("/wallet/list") async def list_wallets(request: Request): """List all wallets linked to the current user.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -504,7 +510,7 @@ class PrivacySettings(BaseModel): @router.get("/privacy-settings") async def get_privacy_settings(request: Request): """Get user privacy settings.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -528,7 +534,8 @@ async def get_privacy_settings(request: Request): @router.patch("/privacy-settings") async def update_privacy_settings(req: PrivacySettings, request: Request): """Update user privacy settings.""" - from app.auth import _save_user, get_current_user + from app.domains.auth import get_current_user + from app.domains.auth.store import _save_user user = await get_current_user(request) if not user: @@ -559,8 +566,10 @@ async def delete_account(req: DeleteAccountRequest, request: Request): Permanently delete user account and all associated data (GDPR compliant). Requires password confirmation. """ - from app.auth import _delete_user, get_current_user, verify_password from app.core.redis import get_redis + from app.domains.auth import get_current_user + from app.domains.auth.passwords import verify_password + from app.domains.auth.store import _delete_user user = await get_current_user(request) if not user: diff --git a/app/routers/bulletin.py b/app/routers/bulletin.py index eed4b6d..24690f8 100644 --- a/app/routers/bulletin.py +++ b/app/routers/bulletin.py @@ -33,7 +33,7 @@ from app.services.supabase_service import ( # noqa: E402 # Lazy auth dependency - avoids importing jose+bcrypt at module load async def _require_public_profile(request: Request): - from app.auth import require_public_profile + from app.domains.auth import require_public_profile return await require_public_profile(request) diff --git a/app/routers/cases_investigators_api.py b/app/routers/cases_investigators_api.py index a524278..b08fd20 100644 --- a/app/routers/cases_investigators_api.py +++ b/app/routers/cases_investigators_api.py @@ -133,7 +133,7 @@ async def create_case(req: CaseCreateRequest, request: Request): # Resolve creator if authed creator = "anonymous" try: - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if user: @@ -199,7 +199,7 @@ async def get_case(case_id: str): async def list_my_cases(request: Request, limit: int = 50): """List cases created by the current user. Auth required.""" try: - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) except Exception: @@ -219,7 +219,7 @@ async def delete_case(case_id: str, request: Request): if not data: raise HTTPException(status_code=404, detail="Case not found") try: - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) except Exception: @@ -261,7 +261,7 @@ async def get_investigator(username: str): async def upsert_investigator(username: str, request: Request): """Update or create investigator profile. Auth required.""" try: - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) except Exception: @@ -312,7 +312,7 @@ async def portfolio_features(request: Request): base_tier = "FREE" try: - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) except Exception: @@ -360,7 +360,7 @@ async def add_addon(req: AddonRequest, request: Request): Returns an x402 challenge for micropayment, or a stripe checkout URL for fiat, or a crypto payment address. """ - from app.auth import get_current_user + from app.domains.auth import get_current_user try: user = await get_current_user(request) diff --git a/app/routers/chat.py b/app/routers/chat.py index 881c6de..91c0f8e 100755 --- a/app/routers/chat.py +++ b/app/routers/chat.py @@ -41,7 +41,7 @@ ai_guard = AIGuard() # ── Auth ── # ── AI Router ── from app.ai_router import router as ai_router # noqa: E402 -from app.auth import get_current_user, require_auth # noqa: E402 +from app.domains.auth import get_current_user, require_auth # noqa: E402 # ── DB path ── DB_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "data", "rmi.db") diff --git a/app/routers/news.py b/app/routers/news.py index 3b7a8df..4a20c74 100644 --- a/app/routers/news.py +++ b/app/routers/news.py @@ -21,7 +21,7 @@ from datetime import UTC, datetime, timedelta from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel -from app.auth import require_public_profile +from app.domains.auth import require_public_profile logger = logging.getLogger(__name__) diff --git a/app/routers/scam_school.py b/app/routers/scam_school.py index 9e37b6c..e37cdaa 100644 --- a/app/routers/scam_school.py +++ b/app/routers/scam_school.py @@ -29,7 +29,7 @@ API_BASE = os.getenv("API_BASE", "https://rugmunch.io") # ── Redis helper ── async def get_current_user(request: Request) -> dict | None: """Extract user from JWT.""" - from app.auth import get_current_user as _get_user + from app.domains.auth import get_current_user as _get_user return await _get_user(request) diff --git a/app/routers/subscription_pricing_api.py b/app/routers/subscription_pricing_api.py index e1a3cad..f797bc7 100644 --- a/app/routers/subscription_pricing_api.py +++ b/app/routers/subscription_pricing_api.py @@ -354,7 +354,7 @@ async def calculate_pricing(tier: str, period: str): @router.get("/subscription") async def get_my_subscription(request: Request): """Get current user's active subscription.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -391,7 +391,7 @@ async def get_my_subscription(request: Request): @router.post("/subscription/create") async def create_subscription(req: SubscriptionCreateRequest, request: Request): """Create a new subscription (Stripe, crypto, or x402).""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -526,7 +526,8 @@ async def create_subscription(req: SubscriptionCreateRequest, request: Request): @router.post("/subscription/crypto/confirm") async def confirm_crypto_payment(req: CryptoPaymentRequest, request: Request): """Confirm a crypto payment (called by user after sending tx, or webhook).""" - from app.auth import _save_user, get_current_user + from app.domains.auth import get_current_user + from app.domains.auth.store import _save_user user = await get_current_user(request) if not user: @@ -570,7 +571,7 @@ async def confirm_crypto_payment(req: CryptoPaymentRequest, request: Request): @router.post("/subscription/cancel") async def cancel_subscription(request: Request): """Cancel subscription at period end.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -596,7 +597,7 @@ async def cancel_subscription(request: Request): @router.post("/subscription/upgrade") async def upgrade_subscription(tier: str, request: Request): """Upgrade to a higher tier.""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user: @@ -636,7 +637,7 @@ async def upgrade_subscription(tier: str, request: Request): @router.post("/webhooks/stripe") async def stripe_webhook(request: Request): """Handle Stripe webhooks for subscription events.""" - from app.auth import _get_user, _save_user + from app.domains.auth.store import _get_user, _save_user payload = await request.body() sig_header = request.headers.get("stripe-signature") @@ -681,7 +682,6 @@ async def stripe_webhook(request: Request): event["data"]["object"] # Mark subscription as past_due # TODO: Implement - pass return {"status": "ok"} @@ -698,7 +698,7 @@ async def list_all_subscriptions( offset: int = 0, ): """List all subscriptions (admin only).""" - from app.auth import get_current_user + from app.domains.auth import get_current_user user = await get_current_user(request) if not user or user.get("role") not in ("ADMIN", "SUPERADMIN"): @@ -732,8 +732,8 @@ async def list_all_subscriptions( @router.get("/admin/revenue") async def get_revenue_stats(request: Request): """Get revenue statistics (admin only).""" - from app.auth import get_current_user from app.core.redis import get_redis + from app.domains.auth import get_current_user user = await get_current_user(request) if not user or user.get("role") not in ("ADMIN", "SUPERADMIN"): diff --git a/app/routers/x402_dashboard.py b/app/routers/x402_dashboard.py index 42fa60f..07e3101 100644 --- a/app/routers/x402_dashboard.py +++ b/app/routers/x402_dashboard.py @@ -53,7 +53,7 @@ def _get_redis(): async def _get_redis_async(): """Async Redis client for dashboard queries.""" try: - from app.auth import get_redis + from app.core.redis import get_redis r = await get_redis() return r diff --git a/app/routers/x402_middleware.py b/app/routers/x402_middleware.py index fc650ec..a2c9d96 100644 --- a/app/routers/x402_middleware.py +++ b/app/routers/x402_middleware.py @@ -338,7 +338,7 @@ async def verify_receipt_public(receipt_id: str): Returns: receipt data + on-chain proof + verification status. """ try: - from app.auth import get_redis + from app.core.redis import get_redis r = await get_redis() @@ -404,7 +404,7 @@ async def payment_ledger( No wallet addresses, no PII. """ try: - from app.auth import get_redis + from app.core.redis import get_redis r = await get_redis() @@ -471,7 +471,7 @@ async def transparency_dashboard(hours: int = 24): No sensitive data - all aggregated and anonymized. """ try: - from app.auth import get_redis + from app.core.redis import get_redis r = await get_redis() @@ -619,7 +619,7 @@ async def verify_payment_endpoint( # Store verification result for audit trail if result["verified"]: try: - from app.auth import get_redis + from app.core.redis import get_redis r = await get_redis() verify_key = f"x402:verify:{result.get('settlement_id') or str(time.time())}" @@ -709,7 +709,7 @@ async def store_receipt(request: Request): amount_float = 0.0 # Store in Redis (sync - app.auth.get_redis returns a sync client) - from app.auth import get_redis as _get_redis + from app.core.redis import get_redis as _get_redis r = _get_redis() @@ -804,7 +804,7 @@ async def fallback_status(): Shows which fallback layers are working and which are failing. """ try: - from app.auth import get_redis + from app.core.redis import get_redis r = await get_redis() diff --git a/app/routers/x402_settle_api.py b/app/routers/x402_settle_api.py index 2bb8d5d..a857ab9 100644 --- a/app/routers/x402_settle_api.py +++ b/app/routers/x402_settle_api.py @@ -119,7 +119,7 @@ async def settle_x402(req: SettleRequest, request: Request): Then activates the subscription / add-on for the user. """ - from app.auth import get_current_user + from app.domains.auth import get_current_user try: user = await get_current_user(request) @@ -187,7 +187,7 @@ async def settle_x402(req: SettleRequest, request: Request): # Also write the user_metadata flag so /api/v1/portfolio/features picks it up try: - from app.auth import _save_user + from app.domains.auth.store import _save_user md = (user.get("user_metadata") or {}).copy() if addon == "PORTFOLIO_PRO": diff --git a/app/scan_rate_limiter.py b/app/scan_rate_limiter.py index 1f87a4b..b80bef7 100644 --- a/app/scan_rate_limiter.py +++ b/app/scan_rate_limiter.py @@ -350,7 +350,7 @@ def get_identity(request) -> tuple[str, str]: auth = request.headers.get("Authorization", "") if hasattr(request, "headers") else "" if auth.startswith("Bearer "): try: - from app.auth import _verify_jwt + from app.domains.auth.jwt import _verify_jwt payload = _verify_jwt(auth[7:]) if payload: From 3571f296283eac75527f4c1399784086d2f9adee Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 15:05:22 +0700 Subject: [PATCH 08/12] refactor(telegram): move config/db/commands to app.domains.telegram.rugmunchbot (P4.6 cont) --- app/contract_deepscan.py | 2 +- app/core/agent_memory.py | 3 +- app/degen_security_scanner.py | 2 +- app/domains/telegram/rugmunchbot/__init__.py | 74 ++++++------ app/domains/telegram/rugmunchbot/bot.py | 110 +++++++++--------- .../telegram/rugmunchbot}/commands/ai.py | 0 .../rugmunchbot}/commands/trending.py | 0 .../telegram/rugmunchbot}/config.py | 0 .../telegram/rugmunchbot}/db.py | 0 app/telegram_bot/bot.py | 78 ++++++------- 10 files changed, 132 insertions(+), 137 deletions(-) rename app/{telegram_bot => domains/telegram/rugmunchbot}/commands/ai.py (100%) rename app/{telegram_bot => domains/telegram/rugmunchbot}/commands/trending.py (100%) rename app/{telegram_bot => domains/telegram/rugmunchbot}/config.py (100%) rename app/{telegram_bot => domains/telegram/rugmunchbot}/db.py (100%) diff --git a/app/contract_deepscan.py b/app/contract_deepscan.py index 98b71de..127b407 100644 --- a/app/contract_deepscan.py +++ b/app/contract_deepscan.py @@ -14,7 +14,7 @@ import tempfile from dataclasses import asdict, dataclass from typing import Any -from app.telegram_bot.requirements import httpx +import httpx logger = logging.getLogger(__name__) diff --git a/app/core/agent_memory.py b/app/core/agent_memory.py index bab72e8..045ac76 100644 --- a/app/core/agent_memory.py +++ b/app/core/agent_memory.py @@ -4,10 +4,9 @@ Enables agents to remember past interactions across sessions.""" import os from datetime import UTC, datetime +import httpx from fastapi import APIRouter -from app.telegram_bot.requirements import httpx - MEMGRAPH_URI = os.getenv("MEMGRAPH_URI", "bolt://localhost:7687") MEMGRAPH_USER = os.getenv("MEMGRAPH_USER", "") MEMGRAPH_PASS = os.getenv("MEMGRAPH_PASSWORD", "") diff --git a/app/degen_security_scanner.py b/app/degen_security_scanner.py index 3c25d0b..cc26741 100644 --- a/app/degen_security_scanner.py +++ b/app/degen_security_scanner.py @@ -1,4 +1,4 @@ -from app.telegram_bot.requirements import httpx +import httpx #!/usr/bin/env python3 """ diff --git a/app/domains/telegram/rugmunchbot/__init__.py b/app/domains/telegram/rugmunchbot/__init__.py index a0e73c0..0106bc5 100644 --- a/app/domains/telegram/rugmunchbot/__init__.py +++ b/app/domains/telegram/rugmunchbot/__init__.py @@ -8,48 +8,48 @@ app.telegram_bot.bot on 2026-07-07). """ from app.domains.telegram.rugmunchbot.bot import ( # noqa: F401 RugMunchBot, - main, - cmd_start, - cmd_help, - cmd_scan, - cmd_wallet, - cmd_ta, - cmd_compare, - cmd_rugcheck, - cmd_trending, - cmd_news, - cmd_account, - cmd_pricing, - cmd_topup, - cmd_scamschool, - cmd_refer, - cmd_watchlist, - cmd_watch, - cmd_unwatch, - cmd_alerts, - cmd_admin_stats, - cmd_admin_set_tier, - cmd_admin_broadcast, - cmd_admin_ban, - handle_message, - handle_inline, - handle_callback, - precheckout, - successful_payment, - error_handler, - setup_bot_profile, - daily_scam_tip, - scan_token, analyze_wallet, + check_spam, + cmd_account, + cmd_admin_ban, + cmd_admin_broadcast, + cmd_admin_set_tier, + cmd_admin_stats, + cmd_alerts, + cmd_compare, + cmd_help, + cmd_news, + cmd_pricing, + cmd_refer, + cmd_rugcheck, + cmd_scamschool, + cmd_scan, + cmd_start, + cmd_ta, + cmd_topup, + cmd_trending, + cmd_unwatch, + cmd_wallet, + cmd_watch, + cmd_watchlist, + daily_scam_tip, + detect_chain, + error_handler, + footer_links, format_scan_report, format_wallet_report, - check_spam, - is_owner, + handle_callback, + handle_inline, + handle_message, is_evm, + is_owner, is_sol, - detect_chain, - short_addr, + main, main_menu_kb, paywall_text, - footer_links, + precheckout, + scan_token, + setup_bot_profile, + short_addr, + successful_payment, ) diff --git a/app/domains/telegram/rugmunchbot/bot.py b/app/domains/telegram/rugmunchbot/bot.py index 55ed103..fc4cc3c 100644 --- a/app/domains/telegram/rugmunchbot/bot.py +++ b/app/domains/telegram/rugmunchbot/bot.py @@ -7,18 +7,17 @@ Telegram Stars payments, DEX ref revenue, referral system, AI chat, inline queries, scheduled tips, trending tokens, spam protection, website integration, social links, bot profile setup. -Run: cd /root/backend/app/telegram_bot && python3 bot.py +Run: cd /root/backend/app/domains/telegram/rugmunchbot && python3 bot.py """ import asyncio +import contextlib import logging import random import re import sys import time -from datetime import UTC, datetime -from datetime import time as dtime -from pathlib import Path +from datetime import UTC, datetime, time as dtime from urllib.parse import quote import httpx @@ -43,11 +42,8 @@ from telegram.ext import ( ) # ── Local imports ── -sys.path.insert(0, "/srv/work/repos/rmi-backend/app/telegram_bot") -import contextlib - -import db -from config import ( +from app.domains.telegram.rugmunchbot import db +from app.domains.telegram.rugmunchbot.config import ( BACKEND_URL, BOT_DESCRIPTION, BOT_SHORT_DESCRIPTION, @@ -2164,61 +2160,61 @@ _HANDLER_REGISTRY = [ __all__ = [ "RugMunchBot", - "main", - "cmd_start", - "cmd_help", - "cmd_scan", - "cmd_wallet", - "cmd_ta", - "cmd_compare", - "cmd_quick", - "cmd_rugcheck", - "cmd_trending", - "cmd_news", - "cmd_account", - "cmd_pricing", - "cmd_topup", - "cmd_scamschool", - "cmd_refer", - "cmd_watchlist", - "cmd_watch", - "cmd_unwatch", - "cmd_alerts", - "cmd_admin_stats", - "cmd_admin_set_tier", - "cmd_admin_broadcast", - "cmd_admin_ban", - "handle_message", - "handle_inline", - "handle_callback", - "precheckout", - "successful_payment", - "error_handler", - "setup_bot_profile", - "daily_scam_tip", - "scan_token", "analyze_wallet", + "back_kb", + "check_spam", + "cmd_account", + "cmd_admin_ban", + "cmd_admin_broadcast", + "cmd_admin_set_tier", + "cmd_admin_stats", + "cmd_alerts", + "cmd_compare", + "cmd_help", + "cmd_news", + "cmd_pricing", + "cmd_quick", + "cmd_refer", + "cmd_rugcheck", + "cmd_scamschool", + "cmd_scan", + "cmd_start", + "cmd_ta", + "cmd_topup", + "cmd_trending", + "cmd_unwatch", + "cmd_wallet", + "cmd_watch", + "cmd_watchlist", + "daily_scam_tip", + "detect_chain", + "dex_buttons", + "error_handler", + "footer_links", "format_scan_report", "format_wallet_report", - "check_spam", - "is_owner", + "handle_callback", + "handle_inline", + "handle_message", "is_evm", + "is_owner", "is_sol", - "detect_chain", - "short_addr", + "main", "main_menu_kb", - "scan_result_kb", - "pricing_kb", - "topup_kb", - "scamschool_kb", - "back_kb", - "social_kb", - "paywall_text", "paywall_kb", - "social_proof_text", - "footer_links", - "web_scan_button", - "dex_buttons", + "paywall_text", + "precheckout", + "pricing_kb", + "scamschool_kb", + "scan_result_kb", + "scan_token", "sep", + "setup_bot_profile", + "short_addr", + "social_kb", + "social_proof_text", + "successful_payment", "thin_sep", + "topup_kb", + "web_scan_button", ] diff --git a/app/telegram_bot/commands/ai.py b/app/domains/telegram/rugmunchbot/commands/ai.py similarity index 100% rename from app/telegram_bot/commands/ai.py rename to app/domains/telegram/rugmunchbot/commands/ai.py diff --git a/app/telegram_bot/commands/trending.py b/app/domains/telegram/rugmunchbot/commands/trending.py similarity index 100% rename from app/telegram_bot/commands/trending.py rename to app/domains/telegram/rugmunchbot/commands/trending.py diff --git a/app/telegram_bot/config.py b/app/domains/telegram/rugmunchbot/config.py similarity index 100% rename from app/telegram_bot/config.py rename to app/domains/telegram/rugmunchbot/config.py diff --git a/app/telegram_bot/db.py b/app/domains/telegram/rugmunchbot/db.py similarity index 100% rename from app/telegram_bot/db.py rename to app/domains/telegram/rugmunchbot/db.py diff --git a/app/telegram_bot/bot.py b/app/telegram_bot/bot.py index 99ef42c..05dd7ff 100644 --- a/app/telegram_bot/bot.py +++ b/app/telegram_bot/bot.py @@ -1,49 +1,49 @@ -"""Backward-compat shim — moved to app.telegram_bot.rugmunchbot.bot in P3B.""" -from app.domains.telegram.rugmunchbot.bot import * # noqa: F401,F403 +"""Backward-compat shim — moved to app.domains.telegram.rugmunchbot.bot in P4.6.""" +from app.domains.telegram.rugmunchbot.bot import * # noqa: F403 from app.domains.telegram.rugmunchbot.bot import ( # noqa: F401 RugMunchBot, - main, - cmd_start, - cmd_help, - cmd_scan, - cmd_wallet, - cmd_ta, - cmd_compare, - cmd_rugcheck, - cmd_trending, - cmd_news, - cmd_account, - cmd_pricing, - cmd_topup, - cmd_scamschool, - cmd_refer, - cmd_watchlist, - cmd_watch, - cmd_unwatch, - cmd_alerts, - cmd_admin_stats, - cmd_admin_set_tier, - cmd_admin_broadcast, - cmd_admin_ban, - handle_message, - handle_inline, - handle_callback, - precheckout, - successful_payment, - error_handler, - setup_bot_profile, - daily_scam_tip, - scan_token, analyze_wallet, + check_spam, + cmd_account, + cmd_admin_ban, + cmd_admin_broadcast, + cmd_admin_set_tier, + cmd_admin_stats, + cmd_alerts, + cmd_compare, + cmd_help, + cmd_news, + cmd_pricing, + cmd_refer, + cmd_rugcheck, + cmd_scamschool, + cmd_scan, + cmd_start, + cmd_ta, + cmd_topup, + cmd_trending, + cmd_unwatch, + cmd_wallet, + cmd_watch, + cmd_watchlist, + daily_scam_tip, + detect_chain, + error_handler, + footer_links, format_scan_report, format_wallet_report, - check_spam, - is_owner, + handle_callback, + handle_inline, + handle_message, is_evm, + is_owner, is_sol, - detect_chain, - short_addr, + main, main_menu_kb, paywall_text, - footer_links, + precheckout, + scan_token, + setup_bot_profile, + short_addr, + successful_payment, ) From 436bc37767a495a72898026a7691a6f750d0f23d Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 15:12:24 +0700 Subject: [PATCH 09/12] refactor(databus): move app.databus engine files to app.domains.databus (P4.5 cont) --- app/core/lifespan.py | 2 +- app/databus/__init__.py | 90 +++++-------------- app/domains/databus/__init__.py | 69 ++++++++++++++ .../databus/_generated/provider_chains.py | 42 +++++---- app/{ => domains}/databus/access_control.py | 0 app/{ => domains}/databus/ai_mcp_servers.py | 0 app/{ => domains}/databus/api_providers.py | 0 app/{ => domains}/databus/arkham_ws.py | 2 +- .../databus/bitquery_provider.py | 2 +- app/{ => domains}/databus/cache.py | 0 app/{ => domains}/databus/core.py | 14 +-- app/{ => domains}/databus/daily_intel.py | 14 +-- app/{ => domains}/databus/data_quality.py | 2 +- .../databus/dataset_providers.py | 0 .../databus/defillama_provider.py | 0 app/{ => domains}/databus/duckdb_analytics.py | 0 .../databus/eth_labels_provider.py | 0 .../databus/evm_extra_providers.py | 0 app/{ => domains}/databus/free_mcp_servers.py | 0 app/{ => domains}/databus/global_news.py | 0 app/{ => domains}/databus/key_affinity.py | 0 app/{ => domains}/databus/mega_news.py | 0 app/{ => domains}/databus/mega_scraper.py | 0 app/{ => domains}/databus/model_registry.py | 0 app/{ => domains}/databus/news_ai_tools.py | 0 app/{ => domains}/databus/news_intel.py | 0 app/{ => domains}/databus/news_mcp_server.py | 0 app/{ => domains}/databus/news_provider.py | 0 app/{ => domains}/databus/ohlcv_engine.py | 2 +- .../databus/premium_mcp_servers.py | 0 app/{ => domains}/databus/premium_scanner.py | 0 app/{ => domains}/databus/provider_chains.py | 0 app/{ => domains}/databus/provider_core.py | 0 .../databus/provider_implementations.py | 0 app/{ => domains}/databus/providers.py | 0 app/domains/databus/providers/__init__.py | 38 ++++---- app/domains/databus/providers/btc.py | 2 +- app/domains/databus/providers/evm.py | 2 +- app/domains/databus/providers/free_tier.py | 6 +- app/domains/databus/providers/mcp_servers.py | 2 +- app/domains/databus/providers/paid_tier.py | 2 +- app/domains/databus/providers/solana.py | 2 +- app/{ => domains}/databus/pyth_provider.py | 0 app/{ => domains}/databus/rag_ingestion.py | 8 +- app/{ => domains}/databus/rag_provider.py | 2 +- app/{ => domains}/databus/response_schema.py | 0 app/{ => domains}/databus/router.py | 60 ++++++------- app/{ => domains}/databus/rugcharts_intel.py | 4 +- app/{ => domains}/databus/security.py | 0 app/{ => domains}/databus/social.py | 2 +- app/{ => domains}/databus/social_feeds.py | 0 app/{ => domains}/databus/social_intel.py | 8 +- app/{ => domains}/databus/social_scraper.py | 2 +- .../databus/spl_metadata_decoder.py | 1 - app/{ => domains}/databus/token_security.py | 2 +- app/{ => domains}/databus/vault.py | 0 .../databus/volume_authenticity.py | 0 app/{ => domains}/databus/webhooks.py | 0 app/{ => domains}/databus/ws_stream.py | 0 app/{ => domains}/databus/x402_mcp_server.py | 0 app/{ => domains}/databus/x_intel.py | 0 app/domains/token/__init__.py | 2 +- app/domains/token/repository.py | 6 +- app/routers/x402_databus_tools.py | 4 +- databus_warm_cron.py | 4 +- scripts/fix_x402_pipeline.py | 2 +- tests/integration/test_databus.py | 2 +- 67 files changed, 216 insertions(+), 186 deletions(-) create mode 100644 app/domains/databus/__init__.py rename app/{ => domains}/databus/access_control.py (100%) rename app/{ => domains}/databus/ai_mcp_servers.py (100%) rename app/{ => domains}/databus/api_providers.py (100%) rename app/{ => domains}/databus/arkham_ws.py (98%) rename app/{ => domains}/databus/bitquery_provider.py (99%) rename app/{ => domains}/databus/cache.py (100%) rename app/{ => domains}/databus/core.py (98%) rename app/{ => domains}/databus/daily_intel.py (97%) rename app/{ => domains}/databus/data_quality.py (99%) rename app/{ => domains}/databus/dataset_providers.py (100%) rename app/{ => domains}/databus/defillama_provider.py (100%) rename app/{ => domains}/databus/duckdb_analytics.py (100%) rename app/{ => domains}/databus/eth_labels_provider.py (100%) rename app/{ => domains}/databus/evm_extra_providers.py (100%) rename app/{ => domains}/databus/free_mcp_servers.py (100%) rename app/{ => domains}/databus/global_news.py (100%) rename app/{ => domains}/databus/key_affinity.py (100%) rename app/{ => domains}/databus/mega_news.py (100%) rename app/{ => domains}/databus/mega_scraper.py (100%) rename app/{ => domains}/databus/model_registry.py (100%) rename app/{ => domains}/databus/news_ai_tools.py (100%) rename app/{ => domains}/databus/news_intel.py (100%) rename app/{ => domains}/databus/news_mcp_server.py (100%) rename app/{ => domains}/databus/news_provider.py (100%) rename app/{ => domains}/databus/ohlcv_engine.py (99%) rename app/{ => domains}/databus/premium_mcp_servers.py (100%) rename app/{ => domains}/databus/premium_scanner.py (100%) rename app/{ => domains}/databus/provider_chains.py (100%) rename app/{ => domains}/databus/provider_core.py (100%) rename app/{ => domains}/databus/provider_implementations.py (100%) rename app/{ => domains}/databus/providers.py (100%) rename app/{ => domains}/databus/pyth_provider.py (100%) rename app/{ => domains}/databus/rag_ingestion.py (96%) rename app/{ => domains}/databus/rag_provider.py (95%) rename app/{ => domains}/databus/response_schema.py (100%) rename app/{ => domains}/databus/router.py (95%) rename app/{ => domains}/databus/rugcharts_intel.py (99%) rename app/{ => domains}/databus/security.py (100%) rename app/{ => domains}/databus/social.py (99%) rename app/{ => domains}/databus/social_feeds.py (100%) rename app/{ => domains}/databus/social_intel.py (98%) rename app/{ => domains}/databus/social_scraper.py (99%) rename app/{ => domains}/databus/spl_metadata_decoder.py (99%) rename app/{ => domains}/databus/token_security.py (99%) rename app/{ => domains}/databus/vault.py (100%) rename app/{ => domains}/databus/volume_authenticity.py (100%) rename app/{ => domains}/databus/webhooks.py (100%) rename app/{ => domains}/databus/ws_stream.py (100%) rename app/{ => domains}/databus/x402_mcp_server.py (100%) rename app/{ => domains}/databus/x_intel.py (100%) diff --git a/app/core/lifespan.py b/app/core/lifespan.py index da285f9..0d6f608 100644 --- a/app/core/lifespan.py +++ b/app/core/lifespan.py @@ -59,7 +59,7 @@ async def _start_background_tasks(app: FastAPI) -> None: # Cache warmer (passes app instance) try: - from app.databus.core import cache_warm_loop + from app.domains.databus.core import cache_warm_loop asyncio.create_task(cache_warm_loop(app)) logger.info("background_task_started", task="databus_cache_warmer", interval="") diff --git a/app/databus/__init__.py b/app/databus/__init__.py index a528a6d..ab2ca20 100644 --- a/app/databus/__init__.py +++ b/app/databus/__init__.py @@ -1,69 +1,23 @@ +"""databus - DEPRECATED shim. Use app.domains.databus. + +Phase 4.5 of AUDIT-2026-Q3.md moved app/databus/ to app.domains.databus/. +This shim re-exports the canonical public surface so legacy imports like +`from app.databus import databus` keep working. + +MIGRATION: replace `from app.databus import ...` with +`from app.domains.databus import ...`. """ -RMI DataBus - The Single Source of Truth for ALL Data -===================================================== - -Every API call, MCP tool, x402 tool, scanner, and frontend hook routes through here. -No raw HTTP calls to external APIs anywhere else. Period. - -Architecture (request flow): - Request → SecurityGate → CreditGate → CacheLayer → ProviderChain → Result - ↓ ↓ ↓ ↓ ↓ - admin check free-first logic L1→L2→L3 local-first! RAG index - vault keys auto-rotate Redis+R2 OUR data 1st WS broadcast - -What it REPLACES (do NOT use these anymore): - - app/cache_manager.py (RMICache) → databus.cache - - app/caching_shield/unified_layer.py → databus.fetch() - - app/caching_shield/data_fallback.py → databus.fetch() - - app/caching_shield/api_registry.py → databus.key_pool - - app/caching_shield/rate_limiter.py → databus.rate_limiter - - app/arkham_connector.py → databus.providers.arkham - - app/coingecko_connector.py → databus.providers.coingecko - - Direct .env reads for API keys → databus.vault (encrypted in-memory) - -OWN DATA FIRST: - Our crown jewels - Wallet Memory Bank, RAG (17K docs), SENTINEL scanner, - Consensus RPC, Funding Tracer, ClickHouse, Price Consensus, News Network, - Bundle Detection, Label Import (169K+) - these are FAST, FREE, and OURS. - They go FIRST in every fallback chain. External APIs only augment or fill gaps. - -Usage: - from app.databus import databus - - # Simple fetch - auto-selects best provider chain - result = await databus.fetch("token_price", mint="So11111111111111111111111111111111111111111") - - # Explicit chain override - result = await databus.fetch("wallet_labels", address="7EcD...", - chain="local_first") - - # Admin-only data (requires admin key in request) - result = await databus.fetch("arkham_entity", address="0x...", - admin_key="...") - - # Check system health - health = await databus.health() - - # Get capacity report (credits, rate limits, recommendations) - report = databus.capacity_report() -""" - -from app.databus.access_control import AccessController, ConsumerType, access_controller -from app.databus.core import DataBus, databus -from app.databus.key_affinity import KeyAffinitySelector, key_affinity -from app.databus.response_schema import SchemaValidator, schema_validator -from app.databus.social import SocialDataAggregator, XTwitterProvider - -__all__ = [ - "AccessController", - "ConsumerType", - "DataBus", - "KeyAffinitySelector", - "SchemaValidator", - "SocialDataAggregator", - "XTwitterProvider", - "access_controller", - "databus", - "key_affinity", - "schema_validator", -] +from app.domains.databus import * # noqa: F403 +from app.domains.databus import ( # noqa: F401 + AccessController, + ConsumerType, + DataBus, + KeyAffinitySelector, + SchemaValidator, + SocialDataAggregator, + XTwitterProvider, + access_controller, + databus, + key_affinity, + schema_validator, +) diff --git a/app/domains/databus/__init__.py b/app/domains/databus/__init__.py new file mode 100644 index 0000000..f91f0c1 --- /dev/null +++ b/app/domains/databus/__init__.py @@ -0,0 +1,69 @@ +""" +RMI DataBus - The Single Source of Truth for ALL Data +===================================================== + +Every API call, MCP tool, x402 tool, scanner, and frontend hook routes through here. +No raw HTTP calls to external APIs anywhere else. Period. + +Architecture (request flow): + Request → SecurityGate → CreditGate → CacheLayer → ProviderChain → Result + ↓ ↓ ↓ ↓ ↓ + admin check free-first logic L1→L2→L3 local-first! RAG index + vault keys auto-rotate Redis+R2 OUR data 1st WS broadcast + +What it REPLACES (do NOT use these anymore): + - app/cache_manager.py (RMICache) → databus.cache + - app/caching_shield/unified_layer.py → databus.fetch() + - app/caching_shield/data_fallback.py → databus.fetch() + - app/caching_shield/api_registry.py → databus.key_pool + - app/caching_shield/rate_limiter.py → databus.rate_limiter + - app/arkham_connector.py → databus.providers.arkham + - app/coingecko_connector.py → databus.providers.coingecko + - Direct .env reads for API keys → databus.vault (encrypted in-memory) + +OWN DATA FIRST: + Our crown jewels - Wallet Memory Bank, RAG (17K docs), SENTINEL scanner, + Consensus RPC, Funding Tracer, ClickHouse, Price Consensus, News Network, + Bundle Detection, Label Import (169K+) - these are FAST, FREE, and OURS. + They go FIRST in every fallback chain. External APIs only augment or fill gaps. + +Usage: + from app.databus import databus + + # Simple fetch - auto-selects best provider chain + result = await databus.fetch("token_price", mint="So11111111111111111111111111111111111111111") + + # Explicit chain override + result = await databus.fetch("wallet_labels", address="7EcD...", + chain="local_first") + + # Admin-only data (requires admin key in request) + result = await databus.fetch("arkham_entity", address="0x...", + admin_key="...") + + # Check system health + health = await databus.health() + + # Get capacity report (credits, rate limits, recommendations) + report = databus.capacity_report() +""" + +from app.domains.databus.access_control import AccessController, ConsumerType, access_controller +from app.domains.databus.core import DataBus, databus +from app.domains.databus.key_affinity import KeyAffinitySelector, key_affinity +from app.domains.databus.response_schema import SchemaValidator, schema_validator +from app.domains.databus.social import SocialDataAggregator, XTwitterProvider + +__all__ = [ + "AccessController", + "ConsumerType", + "DataBus", + "KeyAffinitySelector", + "SchemaValidator", + "SocialDataAggregator", + "XTwitterProvider", + "access_controller", + "databus", + "key_affinity", + "schema_validator", +] diff --git a/app/domains/databus/_generated/provider_chains.py b/app/domains/databus/_generated/provider_chains.py index 9afb30f..00238b7 100644 --- a/app/domains/databus/_generated/provider_chains.py +++ b/app/domains/databus/_generated/provider_chains.py @@ -8,7 +8,7 @@ Source: app/databus/provider_chains.py (legacy shim). import logging -from app.databus.provider_core import ( +from app.domains.databus.provider_core import ( Provider, ProviderChain, ProviderTier, @@ -17,7 +17,7 @@ from app.databus.provider_core import ( _rate_limiters, _RateLimiter, ) -from app.databus.provider_implementations import ( +from app.domains.databus.provider_implementations import ( _alchemy_token_balances, _alchemy_token_metadata, _arkham_counterparties, @@ -63,7 +63,7 @@ from app.databus.provider_implementations import ( _solana_tracker_trending, _virustotal_url_scan, ) -from app.databus.spl_metadata_decoder import _spl_metadata_decoder_provider +from app.domains.databus.spl_metadata_decoder import _spl_metadata_decoder_provider logger = logging.getLogger("databus.providers.chains") def build_provider_chains() -> dict[str, ProviderChain]: @@ -271,7 +271,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── Bitquery chains (activate free plan at bitquery.io/pricing) ── try: - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider _bq = BitqueryProvider() @@ -758,7 +758,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── PREMIUM SCANNER - 10 high-value detection chains ── try: - from app.databus.premium_scanner import ( + from app.domains.databus.premium_scanner import ( detect_bot_farms, detect_bundles, detect_copy_trading, @@ -842,7 +842,11 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── WEBHOOK SYSTEM ── try: - from app.databus.webhooks import handle_webhook, list_webhooks, setup_webhook # noqa: F401 + from app.domains.databus.webhooks import ( # noqa: F401 + handle_webhook, + list_webhooks, + setup_webhook, + ) chains["webhook_handler"] = ProviderChain( "webhook_handler", @@ -866,7 +870,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── ARKHAM WEBSOCKET - real-time intelligence streaming ── try: - from app.databus.arkham_ws import arkham_ws_subscribe + from app.domains.databus.arkham_ws import arkham_ws_subscribe chains["arkham_ws"] = ProviderChain( "arkham_ws", @@ -890,7 +894,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS: Volume Authenticity (Fake Volume %) ── try: - from app.databus.volume_authenticity import analyze_volume_authenticity + from app.domains.databus.volume_authenticity import analyze_volume_authenticity chains["volume_authenticity"] = ProviderChain( "volume_authenticity", @@ -911,7 +915,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS: OHLCV Engine ── try: - from app.databus.ohlcv_engine import fetch_ohlcv, ingest_trade_data + from app.domains.databus.ohlcv_engine import fetch_ohlcv, ingest_trade_data chains["ohlcv"] = ProviderChain( "ohlcv", @@ -939,7 +943,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS: Token Security Matrix (37+ checks) ── try: - from app.databus.token_security import get_check_matrix_endpoint, run_full_scan + from app.domains.databus.token_security import get_check_matrix_endpoint, run_full_scan chains["token_security"] = ProviderChain( "token_security", @@ -967,8 +971,8 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS INTELLIGENCE - 10 Premium Features ── try: - from app.databus.data_quality import enhanced_token_report, get_tier_comparison - from app.databus.rugcharts_intel import ( + from app.domains.databus.data_quality import enhanced_token_report, get_tier_comparison + from app.domains.databus.rugcharts_intel import ( cross_chain_entity, developer_reputation, holder_health_score, @@ -1140,7 +1144,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── NEWS & MARKET DATA - Free APIs ── try: - from app.databus.news_provider import ( + from app.domains.databus.news_provider import ( get_fear_greed, get_full_news_feed, get_market_brief, @@ -1241,7 +1245,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── NEWS INTELLIGENCE ENGINE - multi-source, quality-scored, sentiment-tagged ── try: - from app.databus.news_intel import ( + from app.domains.databus.news_intel import ( add_comment, add_reaction, aggregate_all_news, @@ -1365,7 +1369,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── X/CT INTELLIGENCE - Crypto Twitter Rundown ── try: - from app.databus.x_intel import fetch_ct_rundown, track_ct_accounts + from app.domains.databus.x_intel import fetch_ct_rundown, track_ct_accounts chains["ct_rundown"] = ProviderChain( "ct_rundown", @@ -1401,8 +1405,8 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── SOCIAL INTELLIGENCE - KOL tracking, shill detection, Daily Intel ── try: - from app.databus.daily_intel import generate_daily_intel - from app.databus.social_intel import ( + from app.domains.databus.daily_intel import generate_daily_intel + from app.domains.databus.social_intel import ( detect_shill_campaigns, get_kol_leaderboard, get_kol_profile, @@ -1525,7 +1529,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── MODEL REGISTRY - Smart free model routing, quality review ── try: - from app.databus.model_registry import ai_call, get_usage_stats, review_content + from app.domains.databus.model_registry import ai_call, get_usage_stats, review_content chains["ai_task"] = ProviderChain( "ai_task", @@ -1581,7 +1585,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RAG INGESTION - nightly indexing, health checks ── try: - from app.databus.rag_ingestion import nightly_rag_index, rag_health_check + from app.domains.databus.rag_ingestion import nightly_rag_index, rag_health_check chains["rag_nightly"] = ProviderChain( "rag_nightly", diff --git a/app/databus/access_control.py b/app/domains/databus/access_control.py similarity index 100% rename from app/databus/access_control.py rename to app/domains/databus/access_control.py diff --git a/app/databus/ai_mcp_servers.py b/app/domains/databus/ai_mcp_servers.py similarity index 100% rename from app/databus/ai_mcp_servers.py rename to app/domains/databus/ai_mcp_servers.py diff --git a/app/databus/api_providers.py b/app/domains/databus/api_providers.py similarity index 100% rename from app/databus/api_providers.py rename to app/domains/databus/api_providers.py diff --git a/app/databus/arkham_ws.py b/app/domains/databus/arkham_ws.py similarity index 98% rename from app/databus/arkham_ws.py rename to app/domains/databus/arkham_ws.py index d642740..bf60fd9 100644 --- a/app/databus/arkham_ws.py +++ b/app/domains/databus/arkham_ws.py @@ -97,7 +97,7 @@ async def broadcast_ws_update(address: str, update: dict): # Push to DataBus WebSocket for frontend subscribers try: - from app.databus.ws_stream import ws_manager + from app.domains.databus.ws_stream import ws_manager await ws_manager.broadcast( "arkham_realtime", diff --git a/app/databus/bitquery_provider.py b/app/domains/databus/bitquery_provider.py similarity index 99% rename from app/databus/bitquery_provider.py rename to app/domains/databus/bitquery_provider.py index 7814832..f19c247 100644 --- a/app/databus/bitquery_provider.py +++ b/app/domains/databus/bitquery_provider.py @@ -36,7 +36,7 @@ import time import httpx -from app.databus.cache import CacheLayer, get_cache +from app.domains.databus.cache import CacheLayer, get_cache logger = logging.getLogger("databus.bitquery") diff --git a/app/databus/cache.py b/app/domains/databus/cache.py similarity index 100% rename from app/databus/cache.py rename to app/domains/databus/cache.py diff --git a/app/databus/core.py b/app/domains/databus/core.py similarity index 98% rename from app/databus/core.py rename to app/domains/databus/core.py index 1637ad0..2b4bbcb 100644 --- a/app/databus/core.py +++ b/app/domains/databus/core.py @@ -32,11 +32,11 @@ import time from collections import defaultdict from typing import Any -from app.databus.access_control import access_controller -from app.databus.cache import CacheLayer, get_cache -from app.databus.providers import ProviderChain, ProviderTier, build_provider_chains -from app.databus.security import security -from app.databus.vault import DataBusVault, get_vault +from app.domains.databus.access_control import access_controller +from app.domains.databus.cache import CacheLayer, get_cache +from app.domains.databus.providers import ProviderChain, ProviderTier, build_provider_chains +from app.domains.databus.security import security +from app.domains.databus.vault import DataBusVault, get_vault logger = logging.getLogger("databus.core") @@ -223,7 +223,7 @@ class DataBus: self.chains = build_provider_chains() # Register Bitquery provider for blockchain data try: - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider self.bitquery = BitqueryProvider(self.cache) logger.info("Bitquery provider registered") @@ -485,7 +485,7 @@ class DataBus: "market_movers", ): try: - from app.databus.ws_stream import databus_ws_broadcast + from app.domains.databus.ws_stream import databus_ws_broadcast await databus_ws_broadcast(data_type, result) except Exception: diff --git a/app/databus/daily_intel.py b/app/domains/databus/daily_intel.py similarity index 97% rename from app/databus/daily_intel.py rename to app/domains/databus/daily_intel.py index 78202b2..9e3035d 100644 --- a/app/databus/daily_intel.py +++ b/app/domains/databus/daily_intel.py @@ -89,7 +89,7 @@ async def _gather_all_data() -> dict: # Market data try: - from app.databus.news_provider import get_market_brief + from app.domains.databus.news_provider import get_market_brief data["market"] = await get_market_brief() except Exception: @@ -97,7 +97,7 @@ async def _gather_all_data() -> dict: # News intel try: - from app.databus.news_intel import aggregate_all_news + from app.domains.databus.news_intel import aggregate_all_news data["news"] = await aggregate_all_news(limit=20) except Exception: @@ -105,7 +105,7 @@ async def _gather_all_data() -> dict: # CT Rundown try: - from app.databus.x_intel import fetch_ct_rundown + from app.domains.databus.x_intel import fetch_ct_rundown data["ct"] = await fetch_ct_rundown(limit=15) except Exception: @@ -113,7 +113,7 @@ async def _gather_all_data() -> dict: # Social metrics try: - from app.databus.social_intel import get_social_metrics + from app.domains.databus.social_intel import get_social_metrics data["social"] = await get_social_metrics() except Exception: @@ -121,7 +121,7 @@ async def _gather_all_data() -> dict: # Fear & Greed try: - from app.databus.news_provider import get_fear_greed + from app.domains.databus.news_provider import get_fear_greed data["fear_greed"] = await get_fear_greed() except Exception: @@ -129,7 +129,7 @@ async def _gather_all_data() -> dict: # Prediction markets try: - from app.databus.news_provider import get_prediction_markets + from app.domains.databus.news_provider import get_prediction_markets data["prediction_markets"] = await get_prediction_markets(limit=5) except Exception: @@ -234,7 +234,7 @@ async def generate_daily_intel(publish: bool = False, **kw) -> dict | None: Args: publish: If True, publish to Ghost (primary) + X/Telegram (syndication) """ - from app.databus.model_registry import ai_call, review_content + from app.domains.databus.model_registry import ai_call, review_content # ── PHASE 0: Gather all data ── logger.info("Daily Intel: gathering data...") diff --git a/app/databus/data_quality.py b/app/domains/databus/data_quality.py similarity index 99% rename from app/databus/data_quality.py rename to app/domains/databus/data_quality.py index 31caf9a..c50bc0d 100644 --- a/app/databus/data_quality.py +++ b/app/domains/databus/data_quality.py @@ -570,7 +570,7 @@ async def enhanced_token_report(address: str, chain: str = "solana", **kw) -> di # Get the base token report try: - from app.databus.rugcharts_intel import instant_token_report + from app.domains.databus.rugcharts_intel import instant_token_report base_report = await instant_token_report(address, chain, **kw) except Exception as e: diff --git a/app/databus/dataset_providers.py b/app/domains/databus/dataset_providers.py similarity index 100% rename from app/databus/dataset_providers.py rename to app/domains/databus/dataset_providers.py diff --git a/app/databus/defillama_provider.py b/app/domains/databus/defillama_provider.py similarity index 100% rename from app/databus/defillama_provider.py rename to app/domains/databus/defillama_provider.py diff --git a/app/databus/duckdb_analytics.py b/app/domains/databus/duckdb_analytics.py similarity index 100% rename from app/databus/duckdb_analytics.py rename to app/domains/databus/duckdb_analytics.py diff --git a/app/databus/eth_labels_provider.py b/app/domains/databus/eth_labels_provider.py similarity index 100% rename from app/databus/eth_labels_provider.py rename to app/domains/databus/eth_labels_provider.py diff --git a/app/databus/evm_extra_providers.py b/app/domains/databus/evm_extra_providers.py similarity index 100% rename from app/databus/evm_extra_providers.py rename to app/domains/databus/evm_extra_providers.py diff --git a/app/databus/free_mcp_servers.py b/app/domains/databus/free_mcp_servers.py similarity index 100% rename from app/databus/free_mcp_servers.py rename to app/domains/databus/free_mcp_servers.py diff --git a/app/databus/global_news.py b/app/domains/databus/global_news.py similarity index 100% rename from app/databus/global_news.py rename to app/domains/databus/global_news.py diff --git a/app/databus/key_affinity.py b/app/domains/databus/key_affinity.py similarity index 100% rename from app/databus/key_affinity.py rename to app/domains/databus/key_affinity.py diff --git a/app/databus/mega_news.py b/app/domains/databus/mega_news.py similarity index 100% rename from app/databus/mega_news.py rename to app/domains/databus/mega_news.py diff --git a/app/databus/mega_scraper.py b/app/domains/databus/mega_scraper.py similarity index 100% rename from app/databus/mega_scraper.py rename to app/domains/databus/mega_scraper.py diff --git a/app/databus/model_registry.py b/app/domains/databus/model_registry.py similarity index 100% rename from app/databus/model_registry.py rename to app/domains/databus/model_registry.py diff --git a/app/databus/news_ai_tools.py b/app/domains/databus/news_ai_tools.py similarity index 100% rename from app/databus/news_ai_tools.py rename to app/domains/databus/news_ai_tools.py diff --git a/app/databus/news_intel.py b/app/domains/databus/news_intel.py similarity index 100% rename from app/databus/news_intel.py rename to app/domains/databus/news_intel.py diff --git a/app/databus/news_mcp_server.py b/app/domains/databus/news_mcp_server.py similarity index 100% rename from app/databus/news_mcp_server.py rename to app/domains/databus/news_mcp_server.py diff --git a/app/databus/news_provider.py b/app/domains/databus/news_provider.py similarity index 100% rename from app/databus/news_provider.py rename to app/domains/databus/news_provider.py diff --git a/app/databus/ohlcv_engine.py b/app/domains/databus/ohlcv_engine.py similarity index 99% rename from app/databus/ohlcv_engine.py rename to app/domains/databus/ohlcv_engine.py index 4e93fa2..ca504f7 100644 --- a/app/databus/ohlcv_engine.py +++ b/app/domains/databus/ohlcv_engine.py @@ -340,7 +340,7 @@ async def fetch_ohlcv( # Also compute authenticity if we have volume data authenticity = None try: - from app.databus.volume_authenticity import quick_authenticity_score + from app.domains.databus.volume_authenticity import quick_authenticity_score auth = quick_authenticity_score( volume_24h=summary.get("volume_24h", 0), diff --git a/app/databus/premium_mcp_servers.py b/app/domains/databus/premium_mcp_servers.py similarity index 100% rename from app/databus/premium_mcp_servers.py rename to app/domains/databus/premium_mcp_servers.py diff --git a/app/databus/premium_scanner.py b/app/domains/databus/premium_scanner.py similarity index 100% rename from app/databus/premium_scanner.py rename to app/domains/databus/premium_scanner.py diff --git a/app/databus/provider_chains.py b/app/domains/databus/provider_chains.py similarity index 100% rename from app/databus/provider_chains.py rename to app/domains/databus/provider_chains.py diff --git a/app/databus/provider_core.py b/app/domains/databus/provider_core.py similarity index 100% rename from app/databus/provider_core.py rename to app/domains/databus/provider_core.py diff --git a/app/databus/provider_implementations.py b/app/domains/databus/provider_implementations.py similarity index 100% rename from app/databus/provider_implementations.py rename to app/domains/databus/provider_implementations.py diff --git a/app/databus/providers.py b/app/domains/databus/providers.py similarity index 100% rename from app/databus/providers.py rename to app/domains/databus/providers.py diff --git a/app/domains/databus/providers/__init__.py b/app/domains/databus/providers/__init__.py index 2e8976c..3525891 100644 --- a/app/domains/databus/providers/__init__.py +++ b/app/domains/databus/providers/__init__.py @@ -38,7 +38,7 @@ import httpx logger = logging.getLogger("databus.providers") # Import SPL metadata decoder -from app.databus.spl_metadata_decoder import _spl_metadata_decoder_provider # noqa: E402 +from app.domains.databus.spl_metadata_decoder import _spl_metadata_decoder_provider # noqa: E402 class ProviderTier(Enum): @@ -1539,7 +1539,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── Bitquery chains (activate free plan at bitquery.io/pricing) ── try: - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider _bq = BitqueryProvider() @@ -2026,7 +2026,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── PREMIUM SCANNER - 10 high-value detection chains ── try: - from app.databus.premium_scanner import ( + from app.domains.databus.premium_scanner import ( detect_bot_farms, detect_bundles, detect_copy_trading, @@ -2110,7 +2110,11 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── WEBHOOK SYSTEM ── try: - from app.databus.webhooks import handle_webhook, list_webhooks, setup_webhook # noqa: F401 + from app.domains.databus.webhooks import ( # noqa: F401 + handle_webhook, + list_webhooks, + setup_webhook, + ) chains["webhook_handler"] = ProviderChain( "webhook_handler", @@ -2134,7 +2138,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── ARKHAM WEBSOCKET - real-time intelligence streaming ── try: - from app.databus.arkham_ws import arkham_ws_subscribe + from app.domains.databus.arkham_ws import arkham_ws_subscribe chains["arkham_ws"] = ProviderChain( "arkham_ws", @@ -2158,7 +2162,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS: Volume Authenticity (Fake Volume %) ── try: - from app.databus.volume_authenticity import analyze_volume_authenticity + from app.domains.databus.volume_authenticity import analyze_volume_authenticity chains["volume_authenticity"] = ProviderChain( "volume_authenticity", @@ -2179,7 +2183,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS: OHLCV Engine ── try: - from app.databus.ohlcv_engine import fetch_ohlcv, ingest_trade_data + from app.domains.databus.ohlcv_engine import fetch_ohlcv, ingest_trade_data chains["ohlcv"] = ProviderChain( "ohlcv", @@ -2207,7 +2211,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS: Token Security Matrix (37+ checks) ── try: - from app.databus.token_security import get_check_matrix_endpoint, run_full_scan + from app.domains.databus.token_security import get_check_matrix_endpoint, run_full_scan chains["token_security"] = ProviderChain( "token_security", @@ -2235,8 +2239,8 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RUGCHARTS INTELLIGENCE - 10 Premium Features ── try: - from app.databus.data_quality import enhanced_token_report, get_tier_comparison - from app.databus.rugcharts_intel import ( + from app.domains.databus.data_quality import enhanced_token_report, get_tier_comparison + from app.domains.databus.rugcharts_intel import ( cross_chain_entity, developer_reputation, holder_health_score, @@ -2408,7 +2412,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── NEWS & MARKET DATA - Free APIs ── try: - from app.databus.news_provider import ( + from app.domains.databus.news_provider import ( get_fear_greed, get_full_news_feed, get_market_brief, @@ -2509,7 +2513,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── NEWS INTELLIGENCE ENGINE - multi-source, quality-scored, sentiment-tagged ── try: - from app.databus.news_intel import ( + from app.domains.databus.news_intel import ( add_comment, add_reaction, aggregate_all_news, @@ -2633,7 +2637,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── X/CT INTELLIGENCE - Crypto Twitter Rundown ── try: - from app.databus.x_intel import fetch_ct_rundown, track_ct_accounts + from app.domains.databus.x_intel import fetch_ct_rundown, track_ct_accounts chains["ct_rundown"] = ProviderChain( "ct_rundown", @@ -2669,8 +2673,8 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── SOCIAL INTELLIGENCE - KOL tracking, shill detection, Daily Intel ── try: - from app.databus.daily_intel import generate_daily_intel - from app.databus.social_intel import ( + from app.domains.databus.daily_intel import generate_daily_intel + from app.domains.databus.social_intel import ( detect_shill_campaigns, get_kol_leaderboard, get_kol_profile, @@ -2793,7 +2797,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── MODEL REGISTRY - Smart free model routing, quality review ── try: - from app.databus.model_registry import ai_call, get_usage_stats, review_content + from app.domains.databus.model_registry import ai_call, get_usage_stats, review_content chains["ai_task"] = ProviderChain( "ai_task", @@ -2849,7 +2853,7 @@ def build_provider_chains() -> dict[str, ProviderChain]: # ── RAG INGESTION - nightly indexing, health checks ── try: - from app.databus.rag_ingestion import nightly_rag_index, rag_health_check + from app.domains.databus.rag_ingestion import nightly_rag_index, rag_health_check chains["rag_nightly"] = ProviderChain( "rag_nightly", diff --git a/app/domains/databus/providers/btc.py b/app/domains/databus/providers/btc.py index f8a536f..4250657 100644 --- a/app/domains/databus/providers/btc.py +++ b/app/domains/databus/providers/btc.py @@ -8,7 +8,7 @@ Re-exports the Bitcoin-scoped provider functions from Providers: - _blockchair_address, _blockchair_stats """ -from app.databus.providers import ( # noqa: F401 +from app.domains.databus.providers import ( # noqa: F401 _blockchair_address, _blockchair_stats, ) diff --git a/app/domains/databus/providers/evm.py b/app/domains/databus/providers/evm.py index 142870b..c9fb4e2 100644 --- a/app/domains/databus/providers/evm.py +++ b/app/domains/databus/providers/evm.py @@ -12,7 +12,7 @@ Providers: - _moralis_wallet_net_worth - _etherscan_tx_trace """ -from app.databus.providers import ( # noqa: F401 +from app.domains.databus.providers import ( # noqa: F401 _alchemy_token_balances, _alchemy_token_metadata, _etherscan_tx_trace, diff --git a/app/domains/databus/providers/free_tier.py b/app/domains/databus/providers/free_tier.py index 1b7d98a..c631649 100644 --- a/app/domains/databus/providers/free_tier.py +++ b/app/domains/databus/providers/free_tier.py @@ -15,7 +15,8 @@ Providers: - _passthrough_market_overview, _passthrough_trending, _passthrough_news, _passthrough_alerts, _passthrough_scanner, _passthrough_rag """ -from app.databus.providers import ( # noqa: F401 +from app.domains.databus.providers import ( # noqa: F401 + _coindesk_news, _coingecko_price, _defillama_chains, _defillama_tvl, @@ -26,12 +27,11 @@ from app.databus.providers import ( # noqa: F401 _dexscreener_trades, _local_token_price, _local_wallet_labels, + _messari_news, _passthrough_alerts, _passthrough_market_overview, _passthrough_news, _passthrough_rag, _passthrough_scanner, _passthrough_trending, - _coindesk_news, - _messari_news, ) diff --git a/app/domains/databus/providers/mcp_servers.py b/app/domains/databus/providers/mcp_servers.py index 6b7129e..885a773 100644 --- a/app/domains/databus/providers/mcp_servers.py +++ b/app/domains/databus/providers/mcp_servers.py @@ -5,6 +5,6 @@ Phase 3B of AUDIT-2026-Q3.md. Re-exports the MCP bridge provider function from ``app.databus.providers`` for cleaner import paths. """ -from app.databus.providers import ( # noqa: F401 +from app.domains.databus.providers import ( # noqa: F401 _mcp_bridge, ) diff --git a/app/domains/databus/providers/paid_tier.py b/app/domains/databus/providers/paid_tier.py index 8858247..d1e8b8e 100644 --- a/app/domains/databus/providers/paid_tier.py +++ b/app/domains/databus/providers/paid_tier.py @@ -12,7 +12,7 @@ Providers: - _dune_early_buyers - _virustotal_url_scan """ -from app.databus.providers import ( # noqa: F401 +from app.domains.databus.providers import ( # noqa: F401 _arkham_counterparties, _arkham_entity, _arkham_intel_search, diff --git a/app/domains/databus/providers/solana.py b/app/domains/databus/providers/solana.py index 67227b6..05745af 100644 --- a/app/domains/databus/providers/solana.py +++ b/app/domains/databus/providers/solana.py @@ -9,7 +9,7 @@ Providers: - _birdeye_overview, _birdeye_price (premium Solana) - _solana_tracker_price, _solana_tracker_token, _solana_tracker_trending """ -from app.databus.providers import ( # noqa: F401 +from app.domains.databus.providers import ( # noqa: F401 _birdeye_overview, _birdeye_price, _solana_tracker_price, diff --git a/app/databus/pyth_provider.py b/app/domains/databus/pyth_provider.py similarity index 100% rename from app/databus/pyth_provider.py rename to app/domains/databus/pyth_provider.py diff --git a/app/databus/rag_ingestion.py b/app/domains/databus/rag_ingestion.py similarity index 96% rename from app/databus/rag_ingestion.py rename to app/domains/databus/rag_ingestion.py index 12b45c2..ff918fd 100644 --- a/app/databus/rag_ingestion.py +++ b/app/domains/databus/rag_ingestion.py @@ -26,7 +26,7 @@ async def nightly_rag_index(**kw) -> dict: try: # ── 1. Index recent news articles ── - from app.databus.news_intel import aggregate_all_news + from app.domains.databus.news_intel import aggregate_all_news news = await aggregate_all_news(limit=100) news_docs = [] @@ -55,7 +55,7 @@ async def nightly_rag_index(**kw) -> dict: try: # ── 2. Index CT Rundown ── - from app.databus.x_intel import fetch_ct_rundown + from app.domains.databus.x_intel import fetch_ct_rundown ct = await fetch_ct_rundown(limit=30) ct_docs = [] @@ -84,7 +84,7 @@ async def nightly_rag_index(**kw) -> dict: try: # ── 3. Index market data snapshot ── - from app.databus.news_provider import get_fear_greed, get_market_brief + from app.domains.databus.news_provider import get_fear_greed, get_market_brief market = await get_market_brief() fear = await get_fear_greed() @@ -107,7 +107,7 @@ async def nightly_rag_index(**kw) -> dict: try: # ── 4. Index social metrics ── - from app.databus.social_intel import get_social_metrics + from app.domains.databus.social_intel import get_social_metrics social = await get_social_metrics() social_doc = { diff --git a/app/databus/rag_provider.py b/app/domains/databus/rag_provider.py similarity index 95% rename from app/databus/rag_provider.py rename to app/domains/databus/rag_provider.py index 0e411bf..e01644a 100644 --- a/app/databus/rag_provider.py +++ b/app/domains/databus/rag_provider.py @@ -30,7 +30,7 @@ async def _rag_search_provider(**kwargs) -> dict | None: } try: - from app.databus.rag_engine import ai_enrich, hybrid_search + from app.domains.databus.rag_engine import ai_enrich, hybrid_search if isinstance(collections, str): collections = [c.strip() for c in collections.split(",") if c.strip()] diff --git a/app/databus/response_schema.py b/app/domains/databus/response_schema.py similarity index 100% rename from app/databus/response_schema.py rename to app/domains/databus/response_schema.py diff --git a/app/databus/router.py b/app/domains/databus/router.py similarity index 95% rename from app/databus/router.py rename to app/domains/databus/router.py index 135eb65..750d9c5 100644 --- a/app/databus/router.py +++ b/app/domains/databus/router.py @@ -20,8 +20,8 @@ import os from fastapi import APIRouter, HTTPException, Query, Request -from app.databus.core import databus -from app.databus.social import ( +from app.domains.databus.core import databus +from app.domains.databus.social import ( X_DAILY_READ_BUDGET, X_FREE_MONTHLY_READ_LIMIT, ) @@ -133,11 +133,11 @@ async def databus_chains(): async def databus_access_matrix(request: Request): """View the full access control matrix. Admin key required.""" if not _verify_admin(request): - from app.databus.access_control import access_controller + from app.domains.databus.access_control import access_controller # Non-admin sees a limited view - only their tier's access return {"note": "Full matrix requires admin key. Your access depends on your consumer type."} - from app.databus.access_control import access_controller + from app.domains.databus.access_control import access_controller return access_controller.list_access_matrix() @@ -145,7 +145,7 @@ async def databus_access_matrix(request: Request): @router.get("/mcp-scope/{tool_id}") async def databus_mcp_scope(tool_id: str): """Get the data types an MCP tool is authorized to access.""" - from app.databus.access_control import access_controller + from app.domains.databus.access_control import access_controller allowed = access_controller.get_mcp_allowed_types(tool_id) if not allowed: @@ -156,7 +156,7 @@ async def databus_mcp_scope(tool_id: str): @router.get("/x402-scope/{tier}") async def databus_x402_scope(tier: str): """Get the data types an x402 pricing tier can access.""" - from app.databus.access_control import access_controller + from app.domains.databus.access_control import access_controller allowed = access_controller.get_x402_allowed_types(tier) return {"tier": tier, "allowed_types": allowed} @@ -465,7 +465,7 @@ async def databus_schema_validate(request: Request): body = await request.json() data_type = body.get("data_type", "") data = body.get("data", {}) - from app.databus.response_schema import schema_validator + from app.domains.databus.response_schema import schema_validator is_valid, missing = schema_validator.validate(data_type, data) return {"data_type": data_type, "valid": is_valid, "missing_fields": missing} @@ -477,7 +477,7 @@ async def databus_schema_validate(request: Request): @router.get("/social/x/profile/{username}") async def social_x_profile(username: str): """Get X/Twitter user profile - cached 24h.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) result = await agg.x.get_user(username) @@ -489,7 +489,7 @@ async def social_x_profile(username: str): @router.get("/social/x/tweets/{username}") async def social_x_tweets(username: str, count: int = Query(20, ge=1, le=100)): """Get recent tweets from user - cached 15min.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) result = await agg.get_our_tweets(count=count) if username.lower() == "cryptorugmunch" else None @@ -506,7 +506,7 @@ async def social_x_tweets(username: str, count: int = Query(20, ge=1, le=100)): @router.get("/social/x/mentions") async def social_x_mentions(count: int = Query(20, ge=1, le=100)): """Get mentions of @CryptoRugMunch - cached 15min.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) result = await agg.get_our_mentions(count=count) @@ -518,7 +518,7 @@ async def social_x_mentions(count: int = Query(20, ge=1, le=100)): @router.get("/social/x/engagement") async def social_x_engagement(tweet_ids: str = Query(..., description="Comma-separated tweet IDs")): """Get engagement metrics for tweets - cached 1h.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) ids = [tid.strip() for tid in tweet_ids.split(",")[:100]] @@ -529,7 +529,7 @@ async def social_x_engagement(tweet_ids: str = Query(..., description="Comma-sep @router.get("/social/x/search") async def social_x_search(q: str = Query(..., description="Search query"), count: int = Query(10, ge=1, le=100)): """Search X/Twitter - VERY expensive, cached 24h. Use sparingly.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) result = await agg.search_mentions(q, count=count) @@ -541,7 +541,7 @@ async def social_x_search(q: str = Query(..., description="Search query"), count @router.get("/social/kol/{username}") async def social_kol_reputation(username: str): """KOL reputation score - cached 24h.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) return await agg.get_kol_reputation(username) @@ -550,7 +550,7 @@ async def social_kol_reputation(username: str): @router.get("/social/sentiment") async def social_sentiment(username: str = Query("CryptoRugMunch")): """Brand sentiment analysis - cached 1h.""" - from app.databus.social import SocialDataAggregator + from app.domains.databus.social import SocialDataAggregator agg = SocialDataAggregator(databus.cache) return await agg.get_sentiment(username) @@ -559,7 +559,7 @@ async def social_sentiment(username: str = Query("CryptoRugMunch")): @router.get("/social/x/budget") async def social_x_budget(): """Current X API read budget status.""" - from app.databus.social import XTwitterProvider + from app.domains.databus.social import XTwitterProvider provider = XTwitterProvider(databus.cache) return { @@ -577,7 +577,7 @@ X_HANDLE = "CryptoRugMunch" @router.get("/social/x/discover") async def social_x_discover(handle: str = Query(X_HANDLE), limit: int = Query(50, ge=1, le=100)): """Discover tweets via web search - no API needed, works for free.""" - from app.databus.social_scraper import XWebScraper + from app.domains.databus.social_scraper import XWebScraper scraper = XWebScraper(databus.cache) tweets = await scraper.discover_tweets(handle=handle, limit=limit) @@ -587,7 +587,7 @@ async def social_x_discover(handle: str = Query(X_HANDLE), limit: int = Query(50 @router.get("/social/x/engagement-report") async def social_x_engagement_report(handle: str = Query(X_HANDLE)): """Get engagement report for a handle - avg likes, best tweets, posting frequency.""" - from app.databus.social_scraper import XWebScraper + from app.domains.databus.social_scraper import XWebScraper scraper = XWebScraper(databus.cache) report = await scraper.get_engagement_report(handle=handle) @@ -597,7 +597,7 @@ async def social_x_engagement_report(handle: str = Query(X_HANDLE)): @router.get("/social/x/mentions-discover") async def social_x_mentions_discover(handle: str = Query(X_HANDLE), limit: int = Query(20, ge=1, le=50)): """Find tweets mentioning a handle - via web search.""" - from app.databus.social_scraper import XWebScraper + from app.domains.databus.social_scraper import XWebScraper scraper = XWebScraper(databus.cache) mentions = await scraper.get_mentions(handle=handle, limit=limit) @@ -607,7 +607,7 @@ async def social_x_mentions_discover(handle: str = Query(X_HANDLE), limit: int = @router.get("/social/x/trending") async def social_x_trending(): """Get current crypto trending topics from web search.""" - from app.databus.social_scraper import XWebScraper + from app.domains.databus.social_scraper import XWebScraper scraper = XWebScraper(databus.cache) topics = await scraper.get_trending_topics() @@ -620,7 +620,7 @@ async def social_x_trending(): @router.get("/bitquery/health") async def bitquery_health(): """Check Bitquery provider health and billing status.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) return await provider.health() @@ -629,7 +629,7 @@ async def bitquery_health(): @router.get("/bitquery/token-price/{network}/{token_address}") async def bitquery_token_price(network: str, token_address: str): """Get DEX token price from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) result = await provider.get_token_price(network, token_address) @@ -641,7 +641,7 @@ async def bitquery_token_price(network: str, token_address: str): @router.get("/bitquery/holders/{network}/{token_address}") async def bitquery_holders(network: str, token_address: str, limit: int = Query(100, ge=1, le=500)): """Get token holder distribution from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) result = await provider.get_holder_distribution(network, token_address, limit) @@ -653,7 +653,7 @@ async def bitquery_holders(network: str, token_address: str, limit: int = Query( @router.get("/bitquery/tx-trace/{network}/{tx_hash}") async def bitquery_tx_trace(network: str, tx_hash: str): """Get full transaction trace from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) result = await provider.get_transaction_trace(network, tx_hash) @@ -665,7 +665,7 @@ async def bitquery_tx_trace(network: str, tx_hash: str): @router.get("/bitquery/dex-volume/{network}") async def bitquery_dex_volume(network: str, pool: str = Query(None), timeframe: str = Query("24h")): """Get DEX trading volume from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) result = await provider.get_dex_volume(network, pool, timeframe) @@ -683,7 +683,7 @@ async def bitquery_dex_volume(network: str, pool: str = Query(None), timeframe: @router.get("/bitquery/balance/{network}/{address}") async def bitquery_balance(network: str, address: str): """Get address token balances from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) result = await provider.get_address_balance(network, address) @@ -695,7 +695,7 @@ async def bitquery_balance(network: str, address: str): @router.get("/bitquery/cross-chain/{address}") async def bitquery_cross_chain(address: str, networks: str = Query("ethereum,bsc,solana")): """Track token transfers across chains from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) net_list = [n.strip() for n in networks.split(",")] @@ -710,7 +710,7 @@ async def bitquery_contract_events( network: str, contract: str, event: str = Query(None), limit: int = Query(50, ge=1, le=200) ): """Get smart contract events from Bitquery.""" - from app.databus.bitquery_provider import BitqueryProvider + from app.domains.databus.bitquery_provider import BitqueryProvider provider = BitqueryProvider(databus.cache) result = await provider.get_smart_contract_events(network, contract, event, limit) @@ -734,7 +734,7 @@ async def receive_webhook(service: str, request: Request): indexes in RAG, triggers premium scanner, pushes alerts. """ try: - from app.databus.webhooks import handle_webhook + from app.domains.databus.webhooks import handle_webhook except ImportError: raise HTTPException(501, "Webhook system not available") from None @@ -758,7 +758,7 @@ async def receive_webhook(service: str, request: Request): async def list_webhooks_endpoint(): """List all recent webhook events.""" try: - from app.databus.webhooks import list_webhooks + from app.domains.databus.webhooks import list_webhooks return await list_webhooks() except ImportError: @@ -777,7 +777,7 @@ async def setup_webhook_endpoint(service: str, request: Request): } """ try: - from app.databus.webhooks import setup_webhook + from app.domains.databus.webhooks import setup_webhook except ImportError: raise HTTPException(501, "Webhook system not available") from None diff --git a/app/databus/rugcharts_intel.py b/app/domains/databus/rugcharts_intel.py similarity index 99% rename from app/databus/rugcharts_intel.py rename to app/domains/databus/rugcharts_intel.py index 5b2d670..f16ac9d 100644 --- a/app/databus/rugcharts_intel.py +++ b/app/domains/databus/rugcharts_intel.py @@ -1198,7 +1198,7 @@ async def instant_token_report(address: str = "", chain: str = "solana", **kw) - try: # ── Section 1: Security Scan ── try: - from app.databus.token_security import run_full_scan + from app.domains.databus.token_security import run_full_scan security = await run_full_scan(address, chain, api_key=api_key) if security: @@ -1288,7 +1288,7 @@ async def instant_token_report(address: str = "", chain: str = "solana", **kw) - liq = float(kw.get("liquidity_usd", 0)) uw = int(kw.get("unique_wallets", 0)) if vol_24h > 0 or liq > 0: - from app.databus.volume_authenticity import quick_authenticity_score + from app.domains.databus.volume_authenticity import quick_authenticity_score auth = quick_authenticity_score(vol_24h, liq, uw, 0) report["sections"]["volume"] = { diff --git a/app/databus/security.py b/app/domains/databus/security.py similarity index 100% rename from app/databus/security.py rename to app/domains/databus/security.py diff --git a/app/databus/social.py b/app/domains/databus/social.py similarity index 99% rename from app/databus/social.py rename to app/domains/databus/social.py index b4229ba..c57aa78 100644 --- a/app/databus/social.py +++ b/app/domains/databus/social.py @@ -20,7 +20,7 @@ from datetime import UTC, datetime import httpx -from app.databus.cache import CacheLayer +from app.domains.databus.cache import CacheLayer logger = logging.getLogger("databus.social") diff --git a/app/databus/social_feeds.py b/app/domains/databus/social_feeds.py similarity index 100% rename from app/databus/social_feeds.py rename to app/domains/databus/social_feeds.py diff --git a/app/databus/social_intel.py b/app/domains/databus/social_intel.py similarity index 98% rename from app/databus/social_intel.py rename to app/domains/databus/social_intel.py index 5c1cf42..8011e59 100644 --- a/app/databus/social_intel.py +++ b/app/domains/databus/social_intel.py @@ -295,21 +295,21 @@ async def generate_daily_intel(**kw) -> dict: """ # Gather all data sources try: - from app.databus.news_provider import get_market_brief + from app.domains.databus.news_provider import get_market_brief brief = await get_market_brief() except Exception: brief = {} try: - from app.databus.news_intel import aggregate_all_news + from app.domains.databus.news_intel import aggregate_all_news news = await aggregate_all_news(limit=15) except Exception: news = {"articles": []} try: - from app.databus.x_intel import fetch_ct_rundown + from app.domains.databus.x_intel import fetch_ct_rundown ct = await fetch_ct_rundown(limit=10) except Exception: @@ -411,7 +411,7 @@ async def get_social_metrics(**kw) -> dict: """ # Gather data try: - from app.databus.news_intel import aggregate_all_news + from app.domains.databus.news_intel import aggregate_all_news news = await aggregate_all_news(limit=50) except Exception: diff --git a/app/databus/social_scraper.py b/app/domains/databus/social_scraper.py similarity index 99% rename from app/databus/social_scraper.py rename to app/domains/databus/social_scraper.py index 8c87107..ae5304b 100644 --- a/app/databus/social_scraper.py +++ b/app/domains/databus/social_scraper.py @@ -16,7 +16,7 @@ Cron: Every 6 hours, discover new tweets, extract content, update metrics. import logging from datetime import UTC, datetime -from app.databus.cache import CacheLayer, get_cache +from app.domains.databus.cache import CacheLayer, get_cache logger = logging.getLogger("databus.social_scraper") diff --git a/app/databus/spl_metadata_decoder.py b/app/domains/databus/spl_metadata_decoder.py similarity index 99% rename from app/databus/spl_metadata_decoder.py rename to app/domains/databus/spl_metadata_decoder.py index 269f12b..912b75c 100644 --- a/app/databus/spl_metadata_decoder.py +++ b/app/domains/databus/spl_metadata_decoder.py @@ -257,7 +257,6 @@ async def fetch_metaplex_metadata(mint: str, rpc_url: str) -> dict[str, Any] | N # Check if it's off the ed25519 curve (valid PDA) # For simplicity, we'll just use the first one that doesn't have the high bit set # Actually, proper PDA derivation is complex. Let's use a known endpoint or skip if too complex. - pass # Simpler approach: use getProgramAccounts with filters payload = { diff --git a/app/databus/token_security.py b/app/domains/databus/token_security.py similarity index 99% rename from app/databus/token_security.py rename to app/domains/databus/token_security.py index 7365a5b..ea58782 100644 --- a/app/databus/token_security.py +++ b/app/domains/databus/token_security.py @@ -613,7 +613,7 @@ async def run_quick_scan(address: str, chain: str = "ethereum", **kw) -> dict[st tx_count = int(kw.get("tx_count", 0)) if volume_24h > 0 or liquidity > 0: - from app.databus.volume_authenticity import quick_authenticity_score + from app.domains.databus.volume_authenticity import quick_authenticity_score auth = quick_authenticity_score(volume_24h, liquidity, unique_wallets, tx_count) fake_pct = auth.get("fake_volume_pct", 0) diff --git a/app/databus/vault.py b/app/domains/databus/vault.py similarity index 100% rename from app/databus/vault.py rename to app/domains/databus/vault.py diff --git a/app/databus/volume_authenticity.py b/app/domains/databus/volume_authenticity.py similarity index 100% rename from app/databus/volume_authenticity.py rename to app/domains/databus/volume_authenticity.py diff --git a/app/databus/webhooks.py b/app/domains/databus/webhooks.py similarity index 100% rename from app/databus/webhooks.py rename to app/domains/databus/webhooks.py diff --git a/app/databus/ws_stream.py b/app/domains/databus/ws_stream.py similarity index 100% rename from app/databus/ws_stream.py rename to app/domains/databus/ws_stream.py diff --git a/app/databus/x402_mcp_server.py b/app/domains/databus/x402_mcp_server.py similarity index 100% rename from app/databus/x402_mcp_server.py rename to app/domains/databus/x402_mcp_server.py diff --git a/app/databus/x_intel.py b/app/domains/databus/x_intel.py similarity index 100% rename from app/databus/x_intel.py rename to app/domains/databus/x_intel.py diff --git a/app/domains/token/__init__.py b/app/domains/token/__init__.py index 1b9ef6b..44b460f 100644 --- a/app/domains/token/__init__.py +++ b/app/domains/token/__init__.py @@ -8,7 +8,7 @@ from app.core.health import DomainHealth async def _health_check() -> DomainHealth: """Token health: DataBus is importable.""" try: - import app.databus # noqa: F401 + import app.domains.databus # noqa: F401 return DomainHealth( name="token", healthy=True, diff --git a/app/domains/token/repository.py b/app/domains/token/repository.py index e2936df..835b8b8 100644 --- a/app/domains/token/repository.py +++ b/app/domains/token/repository.py @@ -19,7 +19,7 @@ class TokenRepository: async def get_detail(self, address: str, chain: str) -> TokenDetail: """Fetch token metadata + supply.""" try: - from app.databus.client import get_databus_client + from app.domains.databus.client import get_databus_client client = get_databus_client() raw = await client.get_token_info(address, chain=chain) @@ -36,7 +36,7 @@ class TokenRepository: ) -> list[TokenHolder]: """Fetch top holders.""" try: - from app.databus.client import get_databus_client + from app.domains.databus.client import get_databus_client client = get_databus_client() raw = await client.get_token_holders(address, chain=chain, limit=limit) @@ -48,7 +48,7 @@ class TokenRepository: async def get_liquidity(self, address: str, chain: str) -> TokenLiquidity: """Fetch liquidity + pool info.""" try: - from app.databus.client import get_databus_client + from app.domains.databus.client import get_databus_client client = get_databus_client() raw = await client.get_token_liquidity(address, chain=chain) diff --git a/app/routers/x402_databus_tools.py b/app/routers/x402_databus_tools.py index 7e1a138..3c6e953 100644 --- a/app/routers/x402_databus_tools.py +++ b/app/routers/x402_databus_tools.py @@ -1123,7 +1123,7 @@ async def x402_catalog(request: Request): """Full catalog of available x402 DataBus tools with pricing.""" catalog = [] for tool_id, info in sorted(X402_TOOL_PRICING.items(), key=lambda x: (x[1]["category"], x[1]["price_usd"])): - from app.databus.access_control import access_controller + from app.domains.databus.access_control import access_controller allowed_free = access_controller.get_x402_allowed_types("free") allowed_basic = access_controller.get_x402_allowed_types("basic") @@ -1195,7 +1195,7 @@ async def x402_catalog(request: Request): @router.get("/access-matrix") async def x402_access_matrix(): """Show which data types each x402 tier can access.""" - from app.databus.access_control import access_controller + from app.domains.databus.access_control import access_controller return { "free": access_controller.get_x402_allowed_types("free"), diff --git a/databus_warm_cron.py b/databus_warm_cron.py index 578b1df..7718a0e 100755 --- a/databus_warm_cron.py +++ b/databus_warm_cron.py @@ -15,8 +15,8 @@ os.environ["REDIS_PORT"] = "6379" sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from app.databus.cache import get_cache -from app.databus.core import databus +from app.domains.databus.cache import get_cache +from app.domains.databus.core import databus async def warm(): diff --git a/scripts/fix_x402_pipeline.py b/scripts/fix_x402_pipeline.py index ce82339..bdfacaa 100644 --- a/scripts/fix_x402_pipeline.py +++ b/scripts/fix_x402_pipeline.py @@ -11,7 +11,7 @@ import sys sys.path.insert(0, "/app") -from app.databus.providers import build_provider_chains +from app.domains.databus.providers import build_provider_chains from app.routers.x402_databus_tools import X402_TOOL_PRICING chains = build_provider_chains() diff --git a/tests/integration/test_databus.py b/tests/integration/test_databus.py index 701ec1e..cbfd6ed 100644 --- a/tests/integration/test_databus.py +++ b/tests/integration/test_databus.py @@ -80,7 +80,7 @@ class TestDataBusSmoke: @pytest.mark.parametrize("chain", CHAINS) async def test_chain_exists_in_providers(self, chain): """Every chain should exist in build_provider_chains().""" - from app.databus.providers import build_provider_chains + from app.domains.databus.providers import build_provider_chains chains = build_provider_chains() assert chain in chains, f"Chain '{chain}' missing from provider chains" From 0a8c73d99bf6057d98a5e486ede9280d42235124 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 16:43:49 +0700 Subject: [PATCH 10/12] feat(domains): consolidate bulletin, intelligence, markets, admin, referral, mcp + mypy gate - Make app/domains/auth/ and app/core/redis.py mypy-clean under strict. - Add mypy-gate.ini and Makefile mypy-gate target; promote typecheck-gate in CI. - Consolidate domains into app/domains/: bulletin, admin, intelligence, markets. - Extract referral domain incl. DeFi partner DEX ref links; keep Telegram bot wired. - Move app/mcp/ package and app/api/v1/mcp/router into app/domains/mcp/. - Archive dead app/mcp_router.py. --- .github/workflows/ci.yml | 25 +- Makefile | 14 +- .../legacy_2026_07/mcp_router_2026_07.py} | 0 app/admin_backend.py | 1083 +--------------- app/alert_pipeline.py | 374 +----- app/api/v1/mcp/__init__.py | 10 +- app/bulletin_board.py | 918 +------------ app/core/redis.py | 4 +- app/domains/admin/__init__.py | 31 + app/domains/admin/core.py | 1070 ++++++++++++++++ app/domains/auth/__init__.py | 222 ++-- app/domains/auth/jwt.py | 2 +- app/domains/auth/oauth.py | 88 +- app/domains/auth/schemas.py | 4 +- app/domains/auth/store.py | 4 +- app/domains/auth/totp.py | 8 +- app/domains/auth/wallet.py | 2 +- app/domains/billing/x402/tools/integration.py | 2 +- app/domains/bulletin/__init__.py | 35 + app/domains/bulletin/core.py | 902 +++++++++++++ .../databus/provider_implementations.py | 5 +- app/domains/databus/providers/__init__.py | 5 +- app/domains/databus/webhooks.py | 2 +- app/domains/intelligence/__init__.py | 25 + app/domains/intelligence/alert_pipeline.py | 370 ++++++ app/domains/intelligence/intel_pipeline.py | 298 +++++ app/domains/intelligence/meme_intelligence.py | 423 ++++++ .../n8n_intelligence_workflows.py | 438 +++++++ app/domains/intelligence/news_intelligence.py | 165 +++ app/domains/markets/__init__.py | 19 + app/domains/markets/core.py | 1122 ++++++++++++++++ app/domains/mcp/__init__.py | 33 + app/{ => domains}/mcp/manifest.py | 0 app/{ => domains}/mcp/registry.py | 4 +- app/{api/v1 => domains}/mcp/router.py | 2 +- app/{ => domains}/mcp/server.py | 6 +- .../mcp/tools/eth_labels_tool.py | 0 app/{ => domains}/mcp/x402_ai_guard.py | 0 app/{ => domains}/mcp/x402_bundles.py | 0 app/{ => domains}/mcp/x402_mcp_server.py | 0 app/{ => domains}/mcp/x402_tool_manager.py | 0 app/domains/referral/__init__.py | 31 + app/domains/referral/core.py | 77 ++ app/domains/referral/partners.py | 93 ++ app/domains/telegram/rugmunchbot/bot.py | 65 +- app/domains/telegram/rugmunchbot/config.py | 55 +- app/intel_pipeline.py | 302 +---- app/mcp/__init__.py | 42 +- app/meme_intelligence.py | 427 +------ app/mount.py | 2 +- app/n8n_intelligence_workflows.py | 442 +------ app/news_intelligence.py | 169 +-- app/prediction_market_service.py | 1133 +---------------- app/routers/admin_backend.py | 2 +- app/routers/analytics.py | 2 +- app/routers/bulletin_board.py | 8 +- app/routers/prediction_market_router.py | 2 +- app/routers/wallet_manager_v2.py | 2 +- app/routers/x402_mcp_handler.py | 2 +- app/security_defense.py | 2 +- app/services/prediction_market_intel.py | 2 +- mypy-gate.ini | 14 + mypy.ini | 1 + tests/unit/test_tier1_moat_mcp.py | 2 +- tests/unit/test_tier2_eth_labels_mcp.py | 18 +- 65 files changed, 5541 insertions(+), 5069 deletions(-) rename app/{mcp_router.py => _archive/legacy_2026_07/mcp_router_2026_07.py} (100%) create mode 100644 app/domains/admin/__init__.py create mode 100644 app/domains/admin/core.py create mode 100644 app/domains/bulletin/__init__.py create mode 100644 app/domains/bulletin/core.py create mode 100644 app/domains/intelligence/__init__.py create mode 100644 app/domains/intelligence/alert_pipeline.py create mode 100644 app/domains/intelligence/intel_pipeline.py create mode 100644 app/domains/intelligence/meme_intelligence.py create mode 100644 app/domains/intelligence/n8n_intelligence_workflows.py create mode 100644 app/domains/intelligence/news_intelligence.py create mode 100644 app/domains/markets/__init__.py create mode 100644 app/domains/markets/core.py create mode 100644 app/domains/mcp/__init__.py rename app/{ => domains}/mcp/manifest.py (100%) rename app/{ => domains}/mcp/registry.py (98%) rename app/{api/v1 => domains}/mcp/router.py (99%) rename app/{ => domains}/mcp/server.py (99%) rename app/{ => domains}/mcp/tools/eth_labels_tool.py (100%) rename app/{ => domains}/mcp/x402_ai_guard.py (100%) rename app/{ => domains}/mcp/x402_bundles.py (100%) rename app/{ => domains}/mcp/x402_mcp_server.py (100%) rename app/{ => domains}/mcp/x402_tool_manager.py (100%) create mode 100644 app/domains/referral/__init__.py create mode 100644 app/domains/referral/core.py create mode 100644 app/domains/referral/partners.py create mode 100644 mypy-gate.ini diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de522e2..a289bfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,8 @@ on: # # Phase 1 of AUDIT-2026-Q3.md item P1.6: # - CI was 6/8 decorative (|| true / continue-on-error: true on every job) -# - Now: 2 GATING jobs (build + test) must pass for merge -# - 6 INFORMATIONAL jobs (lint-info, typecheck-info, etc.) report but never gate +# - Now: 3 GATING jobs (build + test + typecheck-gate) must pass for merge +# - 6 INFORMATIONAL jobs (lint-info, typecheck-full-info, etc.) report but never gate # - Forgejo .forgejo/workflows/ci.yml remains the authoritative gate jobs: @@ -61,6 +61,21 @@ jobs: 2>&1 | tail -50 # NO || true — failure GATES merge + typecheck-gate: + name: Typecheck scoped gate (gate) + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v2 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install project editable + run: uv pip install --system -e ".[dev]" + - name: Run scoped mypy gate + run: make mypy-gate + # ────────────────────────── INFORMATIONAL JOBS (fire-and-forget) ────────────────────────── lint-info: @@ -79,8 +94,8 @@ jobs: - name: Run ruff lint (informational) run: ruff check . --statistics --output-format=concise 2>&1 | tail -30 || true - typecheck-info: - name: Typecheck (info only) + typecheck-full-info: + name: Typecheck full codebase (info only) if: always() runs-on: ubuntu-latest continue-on-error: true @@ -93,7 +108,7 @@ jobs: - name: Install mypy run: uv pip install --system mypy - name: Run mypy typecheck (informational) - run: mypy app/ --config-file mypy.ini || true + run: make typecheck || true security-info: name: Security (info only) diff --git a/Makefile b/Makefile index 08de881..152b2f9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev build lint format check test typecheck security ci clean +.PHONY: help install dev build lint format check test test-all security ci clean mypy-gate typecheck-gate help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' @@ -27,8 +27,14 @@ check: ## Full check: lint + format + typecheck + test mypy app/ --ignore-missing-imports pytest tests/ -x -q -m "not integration" -typecheck: ## MyPy type check - mypy app/ --ignore-missing-imports +typecheck: ## MyPy type check (full codebase, informational) + mypy app/ --config-file mypy.ini + +mypy-gate: ## MyPy gate for clean modules (authoritative) + mypy app/domains/auth/ app/core/redis.py --config-file mypy-gate.ini + +typecheck-gate: ## Alias for mypy-gate + $(MAKE) mypy-gate test: ## Run tests (non-integration) pytest tests/ -x -q -m "not integration" @@ -43,7 +49,7 @@ security: ## Security audit ci: ## Full CI pipeline ruff check app/ tests/ ruff format --check app/ tests/ - mypy app/ --ignore-missing-imports + $(MAKE) mypy-gate pytest tests/ -x -q -m "not integration" clean: ## Remove build artifacts diff --git a/app/mcp_router.py b/app/_archive/legacy_2026_07/mcp_router_2026_07.py similarity index 100% rename from app/mcp_router.py rename to app/_archive/legacy_2026_07/mcp_router_2026_07.py diff --git a/app/admin_backend.py b/app/admin_backend.py index dfb357e..5e5889d 100644 --- a/app/admin_backend.py +++ b/app/admin_backend.py @@ -1,1070 +1,19 @@ +"""Deprecated shim - re-exports app.domains.admin. + +Migrate imports from `app.admin_backend` to `app.domains.admin`. +This shim will be removed in Phase 3 cleanup. """ -RMI Admin Backend - Complete Administration System -=================================================== -A hardened, feature-rich admin panel for the RugMunch Intelligence Platform. - -Features: - • Role-Based Access Control (RBAC) - superadmin, admin, moderator, viewer - • Audit Logging - every action logged with IP, timestamp, before/after state - • Rate Limiting - per-endpoint, per-user, per-IP limits - • IP Blocking / Allowlisting - ban bad actors by IP - • 2FA Support - TOTP-based two-factor authentication - • Session Management - active sessions, force logout, expiry control - • System Health - real-time metrics, disk, memory, CPU, service status - • User Management - CRUD, ban/unban, role assignment, activity tracking - • Configuration Management - env vars, feature flags, system settings - • Security Dashboard - failed logins, blocked IPs, threat alerts - • Content Management - announcements, blog posts, SEO settings - • Financial Dashboard - x402 revenue, payment tracking, analytics - • API Key Management - rotate, revoke, scope-limited keys - • Webhook Management - configure, test, monitor webhooks - • Backup & Restore - database, config, snapshot management - -Security: - - All endpoints require admin authentication (JWT + role check) - - Audit log of every admin action (immutable, append-only) - - Rate limiting on all admin endpoints (stricter than public) - - IP allowlist for admin access (optional) - - Session timeout and concurrent session limits - - Password policy enforcement for admin accounts - - Automatic lockout after failed login attempts -""" - from __future__ import annotations -import hashlib -import json -import logging -import os -import secrets -import time -from dataclasses import asdict, dataclass -from datetime import datetime, timedelta -from enum import StrEnum -from typing import Any - -from fastapi import HTTPException, Request - -logger = logging.getLogger("rmi_admin_backend") - - -# ── Role Definitions ────────────────────────────────────────── - - -class AdminRole(StrEnum): - """Admin role hierarchy. Higher = more permissions.""" - - SUPERADMIN = "superadmin" # Full access, can manage other admins - ADMIN = "admin" # Full access except admin management - MODERATOR = "moderator" # Content + user management, no system config - VIEWER = "viewer" # Read-only access to dashboards - SUPPORT = "support" # User management only, read-only system - - -# Permission matrix: role -> set of allowed permissions -PERMISSIONS = { - AdminRole.SUPERADMIN: { - "*", # All permissions - }, - AdminRole.ADMIN: { - "dashboard.read", - "users.read", - "users.write", - "users.ban", - "content.read", - "content.write", - "system.read", - "system.write", - "security.read", - "security.write", - "financial.read", - "financial.write", - "api_keys.read", - "api_keys.write", - "webhooks.read", - "webhooks.write", - "backups.read", - "backups.write", - "settings.read", - "settings.write", - "logs.read", - "analytics.read", - "token_deploy.read", - "token_deploy.write", - }, - AdminRole.MODERATOR: { - "dashboard.read", - "users.read", - "users.write", - "users.ban", - "content.read", - "content.write", - "security.read", - "logs.read", - "analytics.read", - }, - AdminRole.VIEWER: { - "dashboard.read", - "users.read", - "content.read", - "system.read", - "security.read", - "financial.read", - "analytics.read", - "logs.read", - }, - AdminRole.SUPPORT: { - "dashboard.read", - "users.read", - "users.write", - "content.read", - "logs.read", - "analytics.read", - }, -} - - -def has_permission(role: AdminRole, permission: str) -> bool: - """Check if a role has a specific permission.""" - if role == AdminRole.SUPERADMIN: - return True - perms = PERMISSIONS.get(role, set()) - return permission in perms or "*" in perms - - -# ── Audit Log ─────────────────────────────────────────────────── - - -@dataclass -class AuditLogEntry: - """Single audit log entry.""" - - entry_id: str - timestamp: str - admin_id: str - admin_email: str - action: str # e.g., "user.ban", "token.deploy", "config.update" - resource_type: str # e.g., "user", "token", "config" - resource_id: str - ip_address: str - user_agent: str - before_state: dict | None = None - after_state: dict | None = None - status: str = "success" # success, failed, denied - reason: str = "" - session_id: str = "" - request_id: str = "" - - def to_dict(self) -> dict: - return asdict(self) - - -class AuditLogger: - """ - Immutable audit logging system. - Stores to Redis (time-series) + file backup + optional Supabase. - """ - - @staticmethod - async def log( - admin_id: str, - admin_email: str, - action: str, - resource_type: str, - resource_id: str, - ip_address: str = "", - user_agent: str = "", - before_state: dict | None = None, - after_state: dict | None = None, - status: str = "success", - reason: str = "", - session_id: str = "", - request_id: str = "", - ) -> bool: - """Log an admin action.""" - entry = AuditLogEntry( - entry_id=f"audit_{int(time.time() * 1000)}_{secrets.token_hex(4)}", - timestamp=datetime.utcnow().isoformat(), - admin_id=admin_id, - admin_email=admin_email, - action=action, - resource_type=resource_type, - resource_id=resource_id, - ip_address=ip_address, - user_agent=user_agent, - before_state=before_state, - after_state=after_state, - status=status, - reason=reason, - session_id=session_id, - request_id=request_id, - ) - - # Write to Redis (time-series list) - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Add to time-series list - key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" - await r.lpush(key, json.dumps(entry.to_dict())) - - # Keep only last 30 days in Redis - await r.ltrim(key, 0, 9999) - - # Add to admin-specific log - await r.lpush(f"audit_log:admin:{admin_id}", json.dumps(entry.to_dict())) - - # Add to action-specific index - await r.sadd(f"audit_log:actions:{action}", entry.entry_id) - - except Exception as e: - logger.error(f"Redis audit log failed: {e}") - - # Write to local file (append-only, immutable) - try: - log_file = f"/var/log/rmi/audit_{datetime.utcnow().strftime('%Y-%m')}.jsonl" - os.makedirs(os.path.dirname(log_file), exist_ok=True) - with open(log_file, "a") as f: - f.write(json.dumps(entry.to_dict()) + "\n") - except Exception as e: - logger.error(f"File audit log failed: {e}") - - # Write to Supabase if available - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("audit_logs").insert(entry.to_dict()).execute() - except Exception as e: - logger.error(f"Supabase audit log failed: {e}") - - return True - - @staticmethod - async def query( - admin_id: str | None = None, - action: str | None = None, - resource_type: str | None = None, - start_date: str | None = None, - end_date: str | None = None, - limit: int = 100, - offset: int = 0, - ) -> list[AuditLogEntry]: - """Query audit logs.""" - entries = [] - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Get today's log - key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" - logs = await r.lrange(key, offset, offset + limit - 1) - - for log in logs: - data = json.loads(log) - - # Filter - if admin_id and data.get("admin_id") != admin_id: - continue - if action and data.get("action") != action: - continue - if resource_type and data.get("resource_type") != resource_type: - continue - - entries.append(AuditLogEntry(**data)) - - except Exception as e: - logger.error(f"Audit query failed: {e}") - - return entries - - -# ── Security Manager ────────────────────────────────────────── - - -class SecurityManager: - """ - Security hardening for admin backend. - - Rate limiting - - IP blocking - - Failed login tracking - - Session management - """ - - # Rate limits: (max_requests, window_seconds) - RATE_LIMITS = { # noqa: RUF012 - "default": (60, 60), # 60/minute - "login": (5, 300), # 5/5min (strict for login) - "admin_write": (30, 60), # 30/minute for write ops - "token_deploy": (10, 3600), # 10/hour for token deploy - "snapshot": (20, 3600), # 20/hour for snapshots - "airdrop": (5, 3600), # 5/hour for airdrops - } - - @staticmethod - async def check_rate_limit( - identifier: str, - limit_type: str = "default", - ) -> dict[str, Any]: - """Check if request is within rate limit.""" - max_req, window = SecurityManager.RATE_LIMITS.get(limit_type, (60, 60)) - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - key = f"rate_limit:{limit_type}:{identifier}" - now = int(time.time()) - - # Clean old entries - await r.zremrangebyscore(key, 0, now - window) - - # Count current requests - count = await r.zcard(key) - - if count >= max_req: - # Get time until reset - oldest = await r.zrange(key, 0, 0, withscores=True) - reset_at = oldest[0][1] + window if oldest else now + window - - return { - "allowed": False, - "remaining": 0, - "reset_at": reset_at, - "limit": max_req, - "window": window, - } - - # Add current request - await r.zadd(key, {str(now): now}) - await r.expire(key, window) - - return { - "allowed": True, - "remaining": max_req - count - 1, - "reset_at": now + window, - "limit": max_req, - "window": window, - } - - except Exception as e: - logger.error(f"Rate limit check failed: {e}") - # Fail open (allow request) if Redis is down - return {"allowed": True, "remaining": -1, "reset_at": 0} - - @staticmethod - async def block_ip(ip: str, reason: str = "", duration_hours: int = 24) -> bool: - """Block an IP address.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - key = f"blocked_ip:{ip}" - data = { - "ip": ip, - "reason": reason, - "blocked_at": datetime.utcnow().isoformat(), - "expires_at": (datetime.utcnow() + timedelta(hours=duration_hours)).isoformat(), - "duration_hours": duration_hours, - } - - await r.setex(key, duration_hours * 3600, json.dumps(data)) - await r.sadd("blocked_ips:all", ip) - - logger.warning(f"IP blocked: {ip} for {duration_hours}h - {reason}") - return True - - except Exception as e: - logger.error(f"IP block failed: {e}") - return False - - @staticmethod - async def unblock_ip(ip: str) -> bool: - """Unblock an IP address.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - await r.delete(f"blocked_ip:{ip}") - await r.srem("blocked_ips:all", ip) - - return True - - except Exception as e: - logger.error(f"IP unblock failed: {e}") - return False - - @staticmethod - async def is_ip_blocked(ip: str) -> dict | None: - """Check if IP is blocked. Returns block info or None.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - data = await r.get(f"blocked_ip:{ip}") - if data: - return json.loads(data) - - return None - - except Exception as e: - logger.error(f"IP block check failed: {e}") - return None - - @staticmethod - async def track_failed_login(identifier: str) -> int: - """Track failed login attempts. Returns current count.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - key = f"failed_logins:{identifier}" - count = await r.incr(key) - await r.expire(key, 3600) # 1 hour window - - # Auto-block after 5 failed attempts - if count >= 5: - ip = identifier.split(":")[-1] if ":" in identifier else identifier - await SecurityManager.block_ip(ip, "Too many failed login attempts", 1) - - return count - - except Exception as e: - logger.error(f"Failed login tracking error: {e}") - return 0 - - @staticmethod - async def reset_failed_login(identifier: str) -> bool: - """Reset failed login counter (on successful login).""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - await r.delete(f"failed_logins:{identifier}") - return True - - except Exception as e: - logger.error(f"Reset failed login error: {e}") - return False - - -# ── Session Manager ─────────────────────────────────────────── - - -class SessionManager: - """ - Admin session management. - - Track active sessions - - Force logout - - Concurrent session limits - - Session expiry - """ - - MAX_CONCURRENT_SESSIONS = 3 - SESSION_TIMEOUT_HOURS = 8 - - @staticmethod - async def create_session( - admin_id: str, - admin_email: str, - role: AdminRole, - ip_address: str, - user_agent: str, - ) -> str: - """Create a new admin session.""" - session_id = f"sess_{secrets.token_urlsafe(16)}" - - session_data = { - "session_id": session_id, - "admin_id": admin_id, - "admin_email": admin_email, - "role": role.value if isinstance(role, AdminRole) else role, - "ip_address": ip_address, - "user_agent": user_agent, - "created_at": datetime.utcnow().isoformat(), - "expires_at": (datetime.utcnow() + timedelta(hours=SessionManager.SESSION_TIMEOUT_HOURS)).isoformat(), - "last_active": datetime.utcnow().isoformat(), - "is_active": True, - } - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Store session - key = f"admin_session:{session_id}" - await r.setex(key, SessionManager.SESSION_TIMEOUT_HOURS * 3600, json.dumps(session_data)) - - # Add to admin's session list - await r.sadd(f"admin_sessions:{admin_id}", session_id) - - # Enforce max concurrent sessions - sessions = await r.smembers(f"admin_sessions:{admin_id}") - if len(sessions) > SessionManager.MAX_CONCURRENT_SESSIONS: - # Remove oldest sessions - sorted_sessions = sorted(sessions) - for old_session in sorted_sessions[: -SessionManager.MAX_CONCURRENT_SESSIONS]: - await SessionManager.destroy_session(old_session) - - return session_id - - except Exception as e: - logger.error(f"Session creation failed: {e}") - return "" - - @staticmethod - async def validate_session(session_id: str) -> dict | None: - """Validate and refresh a session.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - data = await r.get(f"admin_session:{session_id}") - if not data: - return None - - session = json.loads(data) - - # Check expiry - expires = datetime.fromisoformat(session["expires_at"]) - if datetime.utcnow() > expires: - await SessionManager.destroy_session(session_id) - return None - - # Update last active - session["last_active"] = datetime.utcnow().isoformat() - await r.setex( - f"admin_session:{session_id}", - SessionManager.SESSION_TIMEOUT_HOURS * 3600, - json.dumps(session), - ) - - return session - - except Exception as e: - logger.error(f"Session validation failed: {e}") - return None - - @staticmethod - async def destroy_session(session_id: str) -> bool: - """Destroy a session (logout).""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - # Get session data for admin_id - data = await r.get(f"admin_session:{session_id}") - if data: - session = json.loads(data) - admin_id = session.get("admin_id") - if admin_id: - await r.srem(f"admin_sessions:{admin_id}", session_id) - - await r.delete(f"admin_session:{session_id}") - return True - - except Exception as e: - logger.error(f"Session destroy failed: {e}") - return False - - @staticmethod - async def get_active_sessions(admin_id: str) -> list[dict]: - """Get all active sessions for an admin.""" - sessions = [] - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - session_ids = await r.smembers(f"admin_sessions:{admin_id}") - for sid in session_ids: - data = await r.get(f"admin_session:{sid}") - if data: - session = json.loads(data) - # Check if still valid - expires = datetime.fromisoformat(session["expires_at"]) - if datetime.utcnow() < expires: - sessions.append(session) - else: - await r.srem(f"admin_sessions:{admin_id}", sid) - - except Exception as e: - logger.error(f"Get active sessions failed: {e}") - - return sessions - - @staticmethod - async def destroy_all_sessions(admin_id: str) -> int: - """Destroy all sessions for an admin (force logout everywhere).""" - count = 0 - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - session_ids = await r.smembers(f"admin_sessions:{admin_id}") - for sid in session_ids: - await r.delete(f"admin_session:{sid}") - count += 1 - - await r.delete(f"admin_sessions:{admin_id}") - - except Exception as e: - logger.error(f"Destroy all sessions failed: {e}") - - return count - - -# ── Admin User Store ────────────────────────────────────────── - - -class AdminUserStore: - """ - Store and manage admin user accounts. - Uses Redis as primary + Supabase as backup. - """ - - @staticmethod - async def get_admin(admin_id: str) -> dict | None: - """Get admin by ID.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - data = await r.hget("rmi:admins", admin_id) - if data: - return json.loads(data) - - return None - - except Exception as e: - logger.error(f"Get admin failed: {e}") - return None - - @staticmethod - async def get_admin_by_email(email: str) -> dict | None: - """Get admin by email.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - admin_id = await r.hget("rmi:admins:email", email.lower()) - if admin_id: - return await AdminUserStore.get_admin(admin_id) - - return None - - except Exception as e: - logger.error(f"Get admin by email failed: {e}") - return None - - @staticmethod - async def save_admin(admin: dict) -> bool: - """Save admin user.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - await r.hset("rmi:admins", admin["id"], json.dumps(admin)) - await r.hset("rmi:admins:email", admin["email"].lower(), admin["id"]) - - # Also save to Supabase - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("admin_users").upsert(admin).execute() - except Exception: - pass - - return True - - except Exception as e: - logger.error(f"Save admin failed: {e}") - return False - - @staticmethod - async def list_admins() -> list[dict]: - """List all admin users.""" - admins = [] - - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - all_admins = await r.hgetall("rmi:admins") - for data in all_admins.values(): - admins.append(json.loads(data)) - - except Exception as e: - logger.error(f"List admins failed: {e}") - - return admins - - @staticmethod - async def create_admin( - email: str, - password: str, - role: AdminRole = AdminRole.VIEWER, - created_by: str = "", - ) -> dict | None: - """Create a new admin user.""" - from app.domains.auth.passwords import hash_password - - # Check if email already exists - existing = await AdminUserStore.get_admin_by_email(email) - if existing: - return None - - admin_id = hashlib.sha256(email.lower().encode()).hexdigest()[:16] - - admin = { - "id": admin_id, - "email": email, - "password_hash": hash_password(password), - "role": role.value, - "is_active": True, - "created_at": datetime.utcnow().isoformat(), - "created_by": created_by, - "last_login": None, - "login_count": 0, - "two_factor_enabled": False, - "two_factor_secret": None, - "ip_allowlist": [], - "metadata": {}, - } - - await AdminUserStore.save_admin(admin) - - # Remove password hash from returned dict - safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} - return safe_admin - - @staticmethod - async def verify_admin_login(email: str, password: str) -> dict | None: - """Verify admin login credentials.""" - from app.domains.auth.passwords import verify_password - - admin = await AdminUserStore.get_admin_by_email(email) - if not admin: - return None - - if not admin.get("is_active", True): - return None - - if not verify_password(password, admin.get("password_hash", "")): - return None - - # Update last login - admin["last_login"] = datetime.utcnow().isoformat() - admin["login_count"] = admin.get("login_count", 0) + 1 - await AdminUserStore.save_admin(admin) - - # Remove sensitive data - safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} - return safe_admin - - -# ── System Health Monitor ───────────────────────────────────── - - -class SystemHealthMonitor: - """ - Monitor system health and performance. - """ - - @staticmethod - async def get_system_health() -> dict[str, Any]: - """Get comprehensive system health.""" - import psutil - - # CPU - cpu_percent = psutil.cpu_percent(interval=0.5) - cpu_count = psutil.cpu_count() - - # Memory - memory = psutil.virtual_memory() - - # Disk - disk = psutil.disk_usage("/") - - # Network - net_io = psutil.net_io_counters() - - # Process info - backend_pid = os.getpid() - backend_proc = psutil.Process(backend_pid) - - # Check services - services = { - "redis": await SystemHealthMonitor._check_redis(), - "supabase": await SystemHealthMonitor._check_supabase(), - "backend": {"status": "running", "pid": backend_pid}, - } - - return { - "timestamp": datetime.utcnow().isoformat(), - "cpu": { - "percent": cpu_percent, - "count": cpu_count, - "per_cpu": psutil.cpu_percent(percpu=True, interval=0.1), - }, - "memory": { - "total_gb": round(memory.total / (1024**3), 2), - "available_gb": round(memory.available / (1024**3), 2), - "percent": memory.percent, - "used_gb": round(memory.used / (1024**3), 2), - }, - "disk": { - "total_gb": round(disk.total / (1024**3), 2), - "used_gb": round(disk.used / (1024**3), 2), - "free_gb": round(disk.free / (1024**3), 2), - "percent": round(disk.used / disk.total * 100, 1), - }, - "network": { - "bytes_sent_mb": round(net_io.bytes_sent / (1024**2), 2), - "bytes_recv_mb": round(net_io.bytes_recv / (1024**2), 2), - }, - "backend": { - "pid": backend_pid, - "memory_mb": round(backend_proc.memory_info().rss / (1024**2), 2), - "cpu_percent": backend_proc.cpu_percent(interval=0.2), - "threads": backend_proc.num_threads(), - "open_files": len(backend_proc.open_files()), - "connections": len(backend_proc.connections()), - }, - "services": services, - "uptime_seconds": int(time.time() - psutil.boot_time()), - } - - @staticmethod - async def _check_redis() -> dict: - """Check Redis connection.""" - try: - import redis.asyncio as redis_lib - - r = redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - await r.ping() - info = await r.info() - return { - "status": "connected", - "version": info.get("redis_version", "unknown"), - "used_memory_mb": round(info.get("used_memory", 0) / (1024**2), 2), - "connected_clients": info.get("connected_clients", 0), - "total_keys": info.get("db0", {}).get("keys", 0) if isinstance(info.get("db0"), dict) else 0, - } - except Exception as e: - return {"status": "error", "error": str(e)} - - @staticmethod - async def _check_supabase() -> dict: - """Check Supabase connection.""" - try: - supabase_url = os.getenv("SUPABASE_URL") - if not supabase_url: - return {"status": "not_configured"} - - import httpx - - async with httpx.AsyncClient() as client: - r = await client.get(f"{supabase_url}/rest/v1/", timeout=5) - return { - "status": "connected" if r.status_code < 500 else "error", - "url": supabase_url, - "response_code": r.status_code, - } - except Exception as e: - return {"status": "error", "error": str(e)} - - -# ── Convenience: Admin Auth Decorator ───────────────────────── - - -async def require_admin( - request: Request, - required_permission: str = "", - min_role: AdminRole = AdminRole.VIEWER, -) -> dict[str, Any]: - """ - Verify admin authentication and authorization. - - Usage: - admin = await require_admin(request, "users.write", AdminRole.ADMIN) - """ - # Get session token from header - session_token = request.headers.get("X-Admin-Session", "") - if not session_token: - raise HTTPException(status_code=401, detail="Admin session required") - - # Validate session - session = await SessionManager.validate_session(session_token) - if not session: - raise HTTPException(status_code=401, detail="Invalid or expired session") - - # Check IP block - client_ip = request.client.host if request.client else "" - block_info = await SecurityManager.is_ip_blocked(client_ip) - if block_info: - raise HTTPException(status_code=403, detail="IP blocked") - - # Get admin - admin = await AdminUserStore.get_admin(session["admin_id"]) - if not admin or not admin.get("is_active", True): - raise HTTPException(status_code=403, detail="Admin account inactive") - - # Check role level - role = AdminRole(admin.get("role", "viewer")) - role_order = { - AdminRole.VIEWER: 0, - AdminRole.SUPPORT: 1, - AdminRole.MODERATOR: 2, - AdminRole.ADMIN: 3, - AdminRole.SUPERADMIN: 4, - } - if role_order[role] < role_order[min_role]: - raise HTTPException(status_code=403, detail="Insufficient privileges") - - # Check specific permission - if required_permission and not has_permission(role, required_permission): - raise HTTPException(status_code=403, detail="Permission denied") - - # Check rate limit - rate_check = await SecurityManager.check_rate_limit( - f"admin:{admin['id']}", - "admin_write" if required_permission.endswith(".write") else "default", - ) - if not rate_check["allowed"]: - raise HTTPException(status_code=429, detail=f"Rate limit exceeded. Reset at {rate_check['reset_at']}") - - # Log access - await AuditLogger.log( - admin_id=admin["id"], - admin_email=admin["email"], - action="admin.access", - resource_type="endpoint", - resource_id=str(request.url.path), - ip_address=client_ip, - user_agent=request.headers.get("user-agent", ""), - session_id=session_token, - ) - - return { - "admin": admin, - "session": session, - "role": role, - } +from app.domains.admin import ( # noqa: F401 + PERMISSIONS, + AdminRole, + AdminUserStore, + AuditLogEntry, + AuditLogger, + SecurityManager, + SessionManager, + SystemHealthMonitor, + has_permission, + require_admin, +) diff --git a/app/alert_pipeline.py b/app/alert_pipeline.py index f93c7f9..9b4d5a4 100644 --- a/app/alert_pipeline.py +++ b/app/alert_pipeline.py @@ -1,370 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.alert_pipeline. + +Migrate imports from `app.alert_pipeline` to `app.domains.intelligence.alert_pipeline`. +This shim will be removed in Phase 3 cleanup. """ -RMI Alert Pipeline - Real-time threat intelligence from scanners. -================================================================= -Feeds the Live Intel panel, WebSocket streams, and alert endpoints. +from __future__ import annotations -Data sources: - - SENTINEL scanner (risk scores, scam detection) - - GoPlus security API (honeypot, tax, proxy checks) - - DexScreener new pairs (fresh launches to scan) - - Our own RAG scam patterns - - Wallet label cross-references - -Alert flow: - Scanner → alert_pipeline.push_alert() → Redis sorted set + pub/sub - Homepage reads from /api/v1/alerts/recent - Sidebar reads from /api/v1/alerts/count - WebSocket streams to connected clients -""" - -import json -import logging -import os -import time -from datetime import UTC, datetime - -logger = logging.getLogger("alert_pipeline") - -REDIS_HOST = os.getenv("REDIS_HOST", "rmi-redis") -REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) -REDIS_PW = os.getenv("REDIS_PASSWORD", "") - -ALERT_KEY = "rmi:alerts:recent" -ALERT_COUNT_KEY = "rmi:alerts:count:active" -ALERT_MAX = 500 # Keep last 500 alerts - - -async def _get_redis(): - """Get async Redis connection.""" - import redis.asyncio as aioredis - - return aioredis.Redis( - host=REDIS_HOST, - port=REDIS_PORT, - password=REDIS_PW or None, - decode_responses=True, - ) - - -async def get_active_alert_count() -> int: - """Get count of active (unacknowledged) alerts.""" - try: - r = await _get_redis() - count = await r.zcard(ALERT_KEY) - await r.close() - return count - except Exception as e: - logger.debug(f"Alert count error: {e}") - return 0 - - -async def push_alert( - alert_type: str, - title: str, - description: str = "", - severity: str = "high", - chain: str = "unknown", - token: str = "", - token_symbol: str = "", - wallet: str = "", - risk_score: int = 0, - metadata: dict | None = None, -) -> str: - """ - Push a new alert to the pipeline. - Returns alert_id. - """ - alert_id = f"alt_{int(time.time())}_{os.urandom(4).hex()}" - - alert = { - "id": alert_id, - "type": alert_type, - "title": title, - "description": description, - "severity": severity, - "chain": chain, - "token": token, - "token_symbol": token_symbol, - "wallet": wallet, - "risk_score": risk_score, - "acknowledged": False, - "created_at": datetime.now(UTC).isoformat(), - **(metadata or {}), - } - - try: - r = await _get_redis() - score = time.time() - await r.zadd(ALERT_KEY, {json.dumps(alert): score}) - # Trim old alerts - await r.zremrangebyrank(ALERT_KEY, 0, -(ALERT_MAX + 1)) - # Pub/sub for WebSocket streaming - await r.publish("rmi:ws:alerts", json.dumps(alert)) - await r.close() - logger.info(f"Alert pushed: {alert_type} | {title[:60]}") - except Exception as e: - logger.error(f"Failed to push alert: {e}") - - return alert_id - - -async def get_recent_alerts(limit: int = 20, severity: str = "") -> list[dict]: - """Get recent alerts with optional severity filter. Normalizes old/new formats.""" - try: - r = await _get_redis() - raw = await r.zrevrange(ALERT_KEY, 0, limit * 2 - 1) - await r.close() - - alerts = [] - for entry in raw: - try: - alert = json.loads(entry) - - # Normalize old alert format to new - if "title" not in alert: - alert["title"] = alert.get("message", alert.get("event", "Unknown alert")) - if "description" not in alert: - desc_parts = [] - if alert.get("symbol"): - desc_parts.append(f"Token: {alert['symbol']}") - if alert.get("risk_score"): - desc_parts.append(f"Risk: {alert['risk_score']}/100") - flags = alert.get("risk_flags", []) - if flags: - desc_parts.append("; ".join(str(f)[:80] for f in flags[:2])) - alert["description"] = " | ".join(desc_parts) - if "severity" not in alert: - score = alert.get("risk_score", 50) - alert["severity"] = "critical" if score >= 85 else "high" if score >= 65 else "medium" - if "chain" not in alert: - alert["chain"] = alert.get("chain", "unknown") - - if severity and alert.get("severity") != severity: - continue - alerts.append(alert) - if len(alerts) >= limit: - break - except json.JSONDecodeError: - pass - - return alerts - except Exception as e: - logger.error(f"get_recent_alerts error: {e}") - return [] - - -# ── Alert generators - called by cron jobs or on-demand ────────── - - -async def scan_solana_new_pairs(limit: int = 5) -> int: - """ - Scan latest Solana pairs from DexScreener for scam patterns. - Pushes alerts for high-risk tokens. - Returns number of alerts generated. - """ - import httpx - - pushed = 0 - - try: - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.get("https://api.dexscreener.com/latest/dex/search", params={"q": "SOL USDC"}) - if resp.status_code != 200: - return 0 - - data = resp.json() - pairs = data.get("pairs", [])[:limit] - - for pair in pairs: - token_addr = pair.get("baseToken", {}).get("address", "") - token_name = pair.get("baseToken", {}).get("name", "Unknown") - token_symbol = pair.get("baseToken", {}).get("symbol", "???") - liquidity = float(pair.get("liquidity", {}).get("usd", 0) or 0) - volume = float(pair.get("volume", {}).get("h24", 0) or 0) - price_change = float(pair.get("priceChange", {}).get("h24", 0) or 0) - created_at = pair.get("pairCreatedAt", 0) - age_hours = (time.time() - created_at / 1000) / 3600 if created_at else 999 - - # Risk heuristics - risk_flags = [] - - if liquidity < 1000: - risk_flags.append("low_liquidity") - if volume == 0: - risk_flags.append("no_volume") - if price_change < -80: - risk_flags.append(f"crashed_{abs(price_change):.0f}%") - if age_hours < 1 and liquidity < 5000: - risk_flags.append("fresh_launch_low_liq") - if price_change > 500: - risk_flags.append(f"pumped_{price_change:.0f}%") - - if risk_flags: - await push_alert( - alert_type="new_pair_risk", - title=f"{token_symbol}: {' | '.join(risk_flags[:2])}", - description=f"New pair {token_name} ({token_symbol}) on Solana. " - f"Liquidity: ${liquidity:,.0f}, Age: {age_hours:.1f}h, " - f"24h change: {price_change:+.1f}%", - severity="critical" if len(risk_flags) >= 3 else "high", - chain="solana", - token=token_addr, - token_symbol=token_symbol, - risk_score=min(90, len(risk_flags) * 25), - metadata={ - "risk_flags": risk_flags, - "liquidity_usd": liquidity, - "age_hours": age_hours, - }, - ) - pushed += 1 - - return pushed - except Exception as e: - logger.warning(f"Solana scan error: {e}") - return pushed - - -async def scan_known_scams(limit: int = 3) -> int: - """ - Check our RAG known_scams collection for recently added entries. - Pushes alerts for new scam patterns detected. - """ - pushed = 0 - try: - r = await _get_redis() - # Check for recent scam pattern additions - scam_ids = await r.smembers("rag:idx:known_scams") - - recent_count = 0 - for sid in list(scam_ids)[:20]: - doc = await r.get(f"rag:known_scams:{sid}") - if doc: - try: - data = json.loads(doc) - added = data.get("metadata", {}).get("added_at", "") - if added: - age_h = (time.time() - datetime.fromisoformat(added.replace("Z", "+00:00")).timestamp()) / 3600 - if age_h < 24: - recent_count += 1 - except Exception: - pass - - await r.close() - - if recent_count > 0: - await push_alert( - alert_type="scam_pattern_update", - title=f"{recent_count} new scam patterns detected in last 24h", - description="New rug pull, honeypot, or phishing patterns added to the knowledge base.", - severity="high", - chain="multi", - risk_score=85, - ) - pushed += 1 - - return pushed - except Exception as e: - logger.warning(f"Known scams scan error: {e}") - return pushed - - -async def run_alert_scan() -> dict[str, int]: - """ - Run a full alert scan across all sources. - Called by cron job every 15 minutes. - """ - results = {} - - # Scan Solana new pairs - results["solana_pairs"] = await scan_solana_new_pairs(limit=8) - - # Scan known scams - results["known_scams"] = await scan_known_scams() - - total = sum(results.values()) - logger.info(f"Alert scan complete: {total} new alerts ({results})") - return results - - -# ── Seed some initial alerts if Redis is empty ──────────────────── - - -async def seed_initial_alerts(): - """Seed baseline alerts so the system isn't empty on first run.""" - r = await _get_redis() - existing = await r.zcard(ALERT_KEY) - await r.close() - - if existing > 0: - return # Already has alerts - - seeds = [ - ( - "honeypot", - "Honeypot detected on Base: 0xdead...", - "Token has sell restrictions and blacklist. Buyers cannot exit.", - "critical", - "base", - ), - ( - "whale", - "Whale moved 5M USDC to Binance", - "Wallet 0xABCD... transferred $5M USDC to Binance hot wallet. Possible sell pressure.", - "high", - "ethereum", - ), - ( - "rugpull", - "Liquidity removed from $SCAM token", - "100% of liquidity pool withdrawn by deployer. Token is now worthless.", - "critical", - "solana", - ), - ( - "bundler", - "Sniper bundle detected: $NEWLAUNCH", - "Coordinated wallet cluster bought 60% of supply in first block.", - "high", - "solana", - ), - ( - "contract", - "Unverified proxy contract found", - "Token uses upgradeable proxy with unverified implementation. Owner can change logic.", - "high", - "base", - ), - ( - "concentration", - "90% supply held by 3 wallets on $DANGER", - "Extreme holder concentration. Classic rug pull setup.", - "critical", - "ethereum", - ), - ( - "wash_trade", - "Wash trading detected on $FAKEVOL", - "95% of volume is self-trading between linked wallets.", - "high", - "bsc", - ), - ( - "phishing", - "Fake airdrop targeting $BONK holders", - "Phishing site detected posing as official BONK airdrop. Users losing funds.", - "critical", - "solana", - ), - ] - - for alert_type, title, desc, severity, chain in seeds: - await push_alert( - alert_type=alert_type, - title=title, - description=desc, - severity=severity, - chain=chain, - ) - - logger.info(f"Seeded {len(seeds)} initial alerts") +from app.domains.intelligence.alert_pipeline import * # noqa: F403 diff --git a/app/api/v1/mcp/__init__.py b/app/api/v1/mcp/__init__.py index 036000c..9ba91a7 100644 --- a/app/api/v1/mcp/__init__.py +++ b/app/api/v1/mcp/__init__.py @@ -1,4 +1,10 @@ -"""MCP v1 routes.""" -from .router import router +"""Deprecated shim - MCP router moved to app.domains.mcp.router. + +Migrate imports from `app.api.v1.mcp` to `app.domains.mcp`. +This shim will be removed in Phase 3 cleanup. +""" +from __future__ import annotations + +from app.domains.mcp.router import router __all__ = ["router"] diff --git a/app/bulletin_board.py b/app/bulletin_board.py index 724a761..17641a3 100644 --- a/app/bulletin_board.py +++ b/app/bulletin_board.py @@ -1,903 +1,21 @@ +"""Deprecated shim - re-exports app.domains.bulletin. + +Migrate imports from `app.bulletin_board` to `app.domains.bulletin`. +This shim will be removed in Phase 3 cleanup. """ -RMI Bulletin Board Management System -===================================== -A full content management backend for announcements, news, alerts, platform -communications, and community bulletin boards. - -Features: - • Posts - CRUD with rich text, attachments, scheduling, expiry - • Categories - organize by type (news, alert, update, promo, system) - • Targeting - audience segmentation (all, free, premium, admins, specific tiers) - • Moderation - draft/review/published/archived workflow, approval chains - • Pinning - sticky posts, priority ordering - • Analytics - views, clicks, engagement tracking per post - • Comments - threaded discussions on posts (optional) - • Notifications - push/email/Telegram alerts for critical posts - • Scheduling - publish at future date, auto-archive after expiry - • Versioning - track edit history, rollback capability - • Search - full-text search across all posts - • SEO - slug generation, meta tags, OpenGraph - -Security: - - All write operations require admin auth + content.write permission - - Audit log of every content change - - Rate limiting on publish operations - - Content sanitization (strip XSS, validate HTML) -""" - from __future__ import annotations -import html -import json -import logging -import os -import re -import time -from dataclasses import asdict, dataclass, field -from datetime import datetime -from enum import StrEnum -from typing import Any, ClassVar - -logger = logging.getLogger("rmi_bulletin_board") - - -# ── Enums ───────────────────────────────────────────────────── - - -class PostStatus(StrEnum): - DRAFT = "draft" - REVIEW = "review" - PUBLISHED = "published" - ARCHIVED = "archived" - SCHEDULED = "scheduled" - - -class PostCategory(StrEnum): - NEWS = "news" # Platform news - ALERT = "alert" # Security/urgent alerts - UPDATE = "update" # Feature updates - PROMO = "promo" # Promotions/offers - SYSTEM = "system" # System maintenance - COMMUNITY = "community" # Community posts - ANNOUNCEMENT = "announcement" # General announcements - TUTORIAL = "tutorial" # Guides/how-tos - - -class TargetAudience(StrEnum): - ALL = "all" - FREE = "free" - PREMIUM = "premium" - PRO = "pro" - ENTERPRISE = "enterprise" - ADMINS = "admins" - MODERATORS = "moderators" - - -class Priority(StrEnum): - LOW = "low" - NORMAL = "normal" - HIGH = "high" - CRITICAL = "critical" - - -# ── Data Models ───────────────────────────────────────────── - - -@dataclass -class Post: - """Bulletin board post.""" - - post_id: str - title: str - slug: str - content: str - summary: str - category: str - status: str - priority: str - target_audience: str - author_id: str - author_email: str - author_name: str - created_at: str - updated_at: str - published_at: str | None = None - scheduled_at: str | None = None - expires_at: str | None = None - archived_at: str | None = None - pinned: bool = False - pin_order: int = 0 - featured_image: str = "" - attachments: list[dict] = field(default_factory=list) - tags: list[str] = field(default_factory=list) - meta_title: str = "" - meta_description: str = "" - og_image: str = "" - view_count: int = 0 - click_count: int = 0 - engagement_score: float = 0.0 - version: int = 1 - edit_history: list[dict] = field(default_factory=list) - approved_by: str = "" - approved_at: str | None = None - notification_sent: bool = False - allow_comments: bool = False - comments_count: int = 0 - - def to_dict(self) -> dict: - return asdict(self) - - def to_public_dict(self) -> dict: - """Return public-safe version (no internal fields).""" - return { - "post_id": self.post_id, - "title": self.title, - "slug": self.slug, - "content": self.content, - "summary": self.summary, - "category": self.category, - "priority": self.priority, - "author_name": self.author_name, - "created_at": self.created_at, - "published_at": self.published_at, - "expires_at": self.expires_at, - "pinned": self.pinned, - "pin_order": self.pin_order, - "featured_image": self.featured_image, - "attachments": self.attachments, - "tags": self.tags, - "meta_title": self.meta_title, - "meta_description": self.meta_description, - "og_image": self.og_image, - "view_count": self.view_count, - "allow_comments": self.allow_comments, - "comments_count": self.comments_count, - } - - -@dataclass -class Comment: - """Comment on a post.""" - - comment_id: str - post_id: str - author_id: str - author_name: str - author_email: str - content: str - created_at: str - updated_at: str - parent_id: str | None = None - status: str = "approved" # approved, pending, rejected - likes: int = 0 - replies: list[dict] = field(default_factory=list) - - def to_dict(self) -> dict: - return asdict(self) - - -# ── Content Sanitizer ─────────────────────────────────────── - - -class ContentSanitizer: - """Sanitize user-generated content to prevent XSS.""" - - ALLOWED_TAGS: ClassVar[dict] ={ - "p", - "br", - "strong", - "b", - "em", - "i", - "u", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "ul", - "ol", - "li", - "a", - "img", - "blockquote", - "code", - "pre", - "table", - "thead", - "tbody", - "tr", - "td", - "th", - "div", - "span", - "hr", - "sub", - "sup", - "del", - "ins", - } - - ALLOWED_ATTRS: ClassVar[dict] ={ - "a": ["href", "title", "target"], - "img": ["src", "alt", "title", "width", "height"], - "div": ["class"], - "span": ["class"], - "code": ["class"], - "pre": ["class"], - } - - @staticmethod - def sanitize(text: str) -> str: - """Basic HTML sanitization.""" - if not text: - return "" - - # Escape HTML entities first - text = html.escape(text) - - # Then selectively un-escape allowed tags - # This is a simplified approach - in production use bleach or similar - # For now, strip all HTML tags for safety - text = re.sub(r"<[^>]+>", "", text) - - return text.strip() - - @staticmethod - def generate_slug(title: str) -> str: - """Generate URL-friendly slug from title.""" - slug = re.sub(r"[^\w\s-]", "", title.lower()) - slug = re.sub(r"[-\s]+", "-", slug) - return slug[:80] - - @staticmethod - def generate_summary(content: str, max_length: int = 200) -> str: - """Generate summary from content.""" - # Strip HTML - text = re.sub(r"<[^>]+>", "", content) - text = text.replace("\n", " ").strip() - if len(text) > max_length: - text = text[:max_length].rsplit(" ", 1)[0] + "..." - return text - - -# ── Bulletin Board Manager ──────────────────────────────────── - - -class BulletinBoardManager: - """ - Core manager for bulletin board operations. - Uses Redis as primary store + Supabase backup. - """ - - @staticmethod - async def _get_redis(): - import redis.asyncio as redis_lib - - return redis_lib.Redis( - host=os.getenv("REDIS_HOST", "localhost"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASSWORD", ""), - decode_responses=True, - ) - - @staticmethod - async def create_post( - title: str, - content: str, - category: str, - author_id: str, - author_email: str, - author_name: str = "", - priority: str = "normal", - target_audience: str = "all", - status: str = "draft", - featured_image: str = "", - attachments: list[dict] | None = None, - tags: list[str] | None = None, - scheduled_at: str | None = None, - expires_at: str | None = None, - pinned: bool = False, - allow_comments: bool = False, - meta_title: str = "", - meta_description: str = "", - ) -> Post: - """Create a new post.""" - post_id = f"post_{int(time.time())}_{os.urandom(4).hex()}" - slug = ContentSanitizer.generate_slug(title) - summary = ContentSanitizer.generate_summary(content) - now = datetime.utcnow().isoformat() - - # Sanitize content - content = ContentSanitizer.sanitize(content) - - post = Post( - post_id=post_id, - title=title[:200], - slug=slug, - content=content, - summary=summary, - category=category, - status=status, - priority=priority, - target_audience=target_audience, - author_id=author_id, - author_email=author_email, - author_name=author_name or author_email.split("@")[0], - created_at=now, - updated_at=now, - scheduled_at=scheduled_at, - expires_at=expires_at, - pinned=pinned, - featured_image=featured_image, - attachments=attachments or [], - tags=tags or [], - meta_title=meta_title or title[:70], - meta_description=meta_description or summary[:160], - allow_comments=allow_comments, - ) - - r = await BulletinBoardManager._get_redis() - - # Save post - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post.to_dict())) - - # Add to category index - await r.sadd(f"bulletin:category:{category}", post_id) - - # Add to status index - await r.sadd(f"bulletin:status:{status}", post_id) - - # Add to author index - await r.sadd(f"bulletin:author:{author_id}", post_id) - - # Add to audience index - await r.sadd(f"bulletin:audience:{target_audience}", post_id) - - # Add to pinned index if pinned - if pinned: - await r.sadd("bulletin:pinned", post_id) - await r.zadd("bulletin:pinned_order", {post_id: post.pin_order}) - - # Add to scheduled index if scheduled - if scheduled_at and status == "scheduled": - ts = int(datetime.fromisoformat(scheduled_at).timestamp()) - await r.zadd("bulletin:scheduled", {post_id: ts}) - - # Add to search index (simple word index) - words = set(re.findall(r"\w+", title.lower() + " " + content.lower())) - for word in words: - if len(word) > 2: - await r.sadd(f"bulletin:search:{word}", post_id) - - # Save to Supabase - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("bulletin_posts").insert(post.to_dict()).execute() - except Exception as e: - logger.error(f"Supabase bulletin post save failed: {e}") - - return post - - @staticmethod - async def get_post(post_id: str, increment_views: bool = False) -> Post | None: - """Get a post by ID.""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return None - - post_dict = json.loads(data) - - if increment_views and post_dict.get("status") == "published": - post_dict["view_count"] = post_dict.get("view_count", 0) + 1 - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - - return Post(**post_dict) - - @staticmethod - async def get_post_by_slug(slug: str) -> Post | None: - """Get a post by slug.""" - r = await BulletinBoardManager._get_redis() - # Search all posts for matching slug - all_posts = await r.hgetall("rmi:bulletin_posts") - for _post_id, data in all_posts.items(): - post_dict = json.loads(data) - if post_dict.get("slug") == slug: - return Post(**post_dict) - return None - - @staticmethod - async def update_post( - post_id: str, - updates: dict[str, Any], - editor_id: str = "", - editor_email: str = "", - ) -> Post | None: - """Update a post with versioning.""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return None - - post_dict = json.loads(data) - before_state = {k: post_dict.get(k) for k in updates if k in post_dict} - - # Record edit history - edit_entry = { - "edited_at": datetime.utcnow().isoformat(), - "edited_by": editor_id, - "editor_email": editor_email, - "changes": before_state, - "version": post_dict.get("version", 1), - } - - history = post_dict.get("edit_history", []) - history.append(edit_entry) - post_dict["edit_history"] = history - post_dict["version"] = post_dict.get("version", 1) + 1 - post_dict["updated_at"] = datetime.utcnow().isoformat() - - # Apply updates - for key, value in updates.items(): - if key == "content": - value = ContentSanitizer.sanitize(value) - post_dict["summary"] = ContentSanitizer.generate_summary(value) - if key == "title": - post_dict["slug"] = ContentSanitizer.generate_slug(value) - post_dict[key] = value - - # Handle status transitions - old_status = before_state.get("status") - new_status = post_dict.get("status") - if old_status != new_status: - # Update status indexes - if old_status: - await r.srem(f"bulletin:status:{old_status}", post_id) - await r.sadd(f"bulletin:status:{new_status}", post_id) - - if new_status == "published": - post_dict["published_at"] = datetime.utcnow().isoformat() - elif new_status == "archived": - post_dict["archived_at"] = datetime.utcnow().isoformat() - - # Handle pinning changes - if "pinned" in updates: - if updates["pinned"]: - await r.sadd("bulletin:pinned", post_id) - await r.zadd("bulletin:pinned_order", {post_id: post_dict.get("pin_order", 0)}) - else: - await r.srem("bulletin:pinned", post_id) - await r.zrem("bulletin:pinned_order", post_id) - - # Save updated post - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - - # Update Supabase - try: - from supabase import create_client - - supabase_url = os.getenv("SUPABASE_URL") - supabase_key = os.getenv("SUPABASE_SERVICE_KEY") - if supabase_url and supabase_key: - client = create_client(supabase_url, supabase_key) - client.table("bulletin_posts").upsert(post_dict).execute() - except Exception as e: - logger.error(f"Supabase bulletin post update failed: {e}") - - return Post(**post_dict) - - @staticmethod - async def delete_post(post_id: str) -> bool: - """Delete a post (soft delete - move to archive).""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return False - - post_dict = json.loads(data) - post_dict["status"] = "archived" - post_dict["archived_at"] = datetime.utcnow().isoformat() - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - await r.srem("bulletin:status:published", post_id) - await r.srem("bulletin:status:draft", post_id) - await r.srem("bulletin:status:review", post_id) - await r.sadd("bulletin:status:archived", post_id) - await r.srem("bulletin:pinned", post_id) - - return True - - @staticmethod - async def list_posts( - category: str | None = None, - status: str | None = None, - target_audience: str | None = None, - author_id: str | None = None, - pinned_only: bool = False, - search_query: str | None = None, - tags: list[str] | None = None, - limit: int = 50, - offset: int = 0, - sort_by: str = "created_at", - sort_order: str = "desc", - ) -> dict[str, Any]: - """List posts with filtering.""" - r = await BulletinBoardManager._get_redis() - - # Start with all posts or filtered set - if pinned_only: - post_ids = await r.smembers("bulletin:pinned") - elif category: - post_ids = await r.smembers(f"bulletin:category:{category}") - elif status: - post_ids = await r.smembers(f"bulletin:status:{status}") - elif author_id: - post_ids = await r.smembers(f"bulletin:author:{author_id}") - elif target_audience: - post_ids = await r.smembers(f"bulletin:audience:{target_audience}") - elif search_query: - # Search by words - words = re.findall(r"\w+", search_query.lower()) - if words: - sets = [f"bulletin:search:{w}" for w in words if len(w) > 2] - if sets: - post_ids = await r.sinter(sets) - else: - post_ids = set() - else: - post_ids = set() - else: - all_posts = await r.hgetall("rmi:bulletin_posts") - post_ids = set(all_posts.keys()) - - # Apply additional filters - if tags: - tagged_posts = set() - all_posts = await r.hgetall("rmi:bulletin_posts") - for pid, data in all_posts.items(): - post_dict = json.loads(data) - if any(tag in post_dict.get("tags", []) for tag in tags): - tagged_posts.add(pid) - post_ids = post_ids.intersection(tagged_posts) - - # Fetch and sort posts - posts = [] - all_posts = await r.hgetall("rmi:bulletin_posts") - for pid in post_ids: - data = all_posts.get(pid) - if data: - post_dict = json.loads(data) - posts.append(post_dict) - - # Sort - reverse = sort_order == "desc" - posts.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse) - - # Pinned posts first if not pinned_only - if not pinned_only: - pinned_ids = await r.smembers("bulletin:pinned") - posts.sort( - key=lambda x: (x["post_id"] not in pinned_ids, x.get(sort_by, "")), - reverse=not reverse, - ) - - total = len(posts) - posts = posts[offset : offset + limit] - - return { - "posts": posts, - "total": total, - "limit": limit, - "offset": offset, - } - - @staticmethod - async def publish_scheduled() -> list[str]: - """Publish posts that are scheduled for now.""" - r = await BulletinBoardManager._get_redis() - now = int(time.time()) - - # Get scheduled posts that are due - due = await r.zrangebyscore("bulletin:scheduled", 0, now) - published = [] - - for post_id in due: - data = await r.hget("rmi:bulletin_posts", post_id) - if data: - post_dict = json.loads(data) - post_dict["status"] = "published" - post_dict["published_at"] = datetime.utcnow().isoformat() - post_dict["scheduled_at"] = None - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - await r.srem("bulletin:status:scheduled", post_id) - await r.sadd("bulletin:status:published", post_id) - await r.zrem("bulletin:scheduled", post_id) - - published.append(post_id) - - return published - - @staticmethod - async def archive_expired() -> list[str]: - """Archive posts that have expired.""" - r = await BulletinBoardManager._get_redis() - now = datetime.utcnow().isoformat() - - all_posts = await r.hgetall("rmi:bulletin_posts") - archived = [] - - for post_id, data in all_posts.items(): - post_dict = json.loads(data) - if post_dict.get("status") == "published" and post_dict.get("expires_at"): - if post_dict["expires_at"] < now: - post_dict["status"] = "archived" - post_dict["archived_at"] = now - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - await r.srem("bulletin:status:published", post_id) - await r.sadd("bulletin:status:archived", post_id) - await r.srem("bulletin:pinned", post_id) - - archived.append(post_id) - - return archived - - @staticmethod - async def add_comment( - post_id: str, - author_id: str, - author_name: str, - author_email: str, - content: str, - parent_id: str | None = None, - ) -> Comment | None: - """Add a comment to a post.""" - r = await BulletinBoardManager._get_redis() - - # Check if post exists and allows comments - post_data = await r.hget("rmi:bulletin_posts", post_id) - if not post_data: - return None - - post_dict = json.loads(post_data) - if not post_dict.get("allow_comments", False): - return None - - comment_id = f"comment_{int(time.time())}_{os.urandom(4).hex()}" - now = datetime.utcnow().isoformat() - - comment = Comment( - comment_id=comment_id, - post_id=post_id, - author_id=author_id, - author_name=author_name, - author_email=author_email, - content=ContentSanitizer.sanitize(content)[:2000], - created_at=now, - updated_at=now, - parent_id=parent_id, - ) - - await r.hset("rmi:bulletin_comments", comment_id, json.dumps(comment.to_dict())) - await r.sadd(f"bulletin:post_comments:{post_id}", comment_id) - - # Update post comment count - post_dict["comments_count"] = post_dict.get("comments_count", 0) + 1 - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - - return comment - - @staticmethod - async def get_comments(post_id: str, limit: int = 100) -> list[dict]: - """Get comments for a post.""" - r = await BulletinBoardManager._get_redis() - comment_ids = await r.smembers(f"bulletin:post_comments:{post_id}") - - comments = [] - for cid in comment_ids: - data = await r.hget("rmi:bulletin_comments", cid) - if data: - comments.append(json.loads(data)) - - comments.sort(key=lambda x: x.get("created_at", ""), reverse=True) - return comments[:limit] - - @staticmethod - async def get_stats() -> dict[str, Any]: - """Get bulletin board statistics.""" - r = await BulletinBoardManager._get_redis() - - stats = { - "total_posts": await r.hlen("rmi:bulletin_posts") or 0, - "published": await r.scard("bulletin:status:published") or 0, - "drafts": await r.scard("bulletin:status:draft") or 0, - "review": await r.scard("bulletin:status:review") or 0, - "archived": await r.scard("bulletin:status:archived") or 0, - "scheduled": await r.scard("bulletin:status:scheduled") or 0, - "pinned": await r.scard("bulletin:pinned") or 0, - "total_comments": await r.hlen("rmi:bulletin_comments") or 0, - "categories": {}, - } - - # Count by category - for cat in PostCategory: - count = await r.scard(f"bulletin:category:{cat.value}") or 0 - stats["categories"][cat.value] = count - - return stats - - @staticmethod - async def track_engagement(post_id: str, action: str) -> bool: - """Track engagement (view, click, etc.).""" - r = await BulletinBoardManager._get_redis() - data = await r.hget("rmi:bulletin_posts", post_id) - if not data: - return False - - post_dict = json.loads(data) - - if action == "view": - post_dict["view_count"] = post_dict.get("view_count", 0) + 1 - elif action == "click": - post_dict["click_count"] = post_dict.get("click_count", 0) + 1 - - # Calculate engagement score - views = post_dict.get("view_count", 0) - clicks = post_dict.get("click_count", 0) - post_dict["engagement_score"] = round((clicks / max(views, 1)) * 100, 2) - - await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) - return True - - -# ═══════════════════════════════════════════════════════════ -# BADGES & X402 BOT PAYMENTS -# ═══════════════════════════════════════════════════════════ - -BADGES = { - "first_post": { - "name": "First Words", - "icon": "💬", - "desc": "Made your first post", - "tier": "bronze", - }, - "10_posts": {"name": "Chatterbox", "icon": "📢", "desc": "10 posts", "tier": "bronze"}, - "50_posts": {"name": "Board Regular", "icon": "🎙️", "desc": "50 posts", "tier": "silver"}, - "100_posts": { - "name": "Terminally Online", - "icon": "🖥️", - "desc": "100 posts - touch grass", - "tier": "gold", - }, - "10_upvotes": { - "name": "Approved", - "icon": "👍", - "desc": "10 upvotes on a post", - "tier": "bronze", - }, - "50_upvotes": {"name": "Crowd Favorite", "icon": "⭐", "desc": "50 upvotes", "tier": "silver"}, - "100_upvotes": {"name": "Legendary", "icon": "👑", "desc": "100 upvotes", "tier": "gold"}, - "rug_reporter": { - "name": "Rug Detective", - "icon": "🔍", - "desc": "Reported 3 verified rugs", - "tier": "silver", - }, - "scam_buster": { - "name": "Scam Buster", - "icon": "🛡️", - "desc": "10 scam alerts verified", - "tier": "gold", - }, - "honeypot_hunter": { - "name": "Honeypot Hunter", - "icon": "🍯", - "desc": "Found 5 honeypots", - "tier": "silver", - }, - "whale_watcher": { - "name": "Whale Watcher", - "icon": "🐋", - "desc": "Tracked 10 whale moves", - "tier": "silver", - }, - "alpha_caller": { - "name": "Alpha Caller", - "icon": "📈", - "desc": "Called 5 pumps", - "tier": "gold", - }, - "degen": { - "name": "Certified Degen", - "icon": "🎰", - "desc": "Posted in every category", - "tier": "gold", - }, - "rug_survivor": { - "name": "Rug Survivor", - "icon": "💀", - "desc": "Posted about getting rugged", - "tier": "bronze", - }, - "diamond_hands": { - "name": "Diamond Hands", - "icon": "💎", - "desc": "Held through -90%", - "tier": "diamond", - }, - "based": { - "name": "Based", - "icon": "🧠", - "desc": "Called a 10x before it happened", - "tier": "diamond", - }, - "bot_verified": { - "name": "Verified Bot", - "icon": "🤖", - "desc": "Registered x402 bot", - "tier": "silver", - }, - "bot_pro": {"name": "Bot Pro", "icon": "⚡", "desc": "100+ API calls", "tier": "gold"}, -} -BADGE_TIERS = {"bronze": "#CD7F32", "silver": "#C0C0C0", "gold": "#FFD700", "diamond": "#B9F2FF"} - - -async def get_user_badges(user_id: str) -> list: - try: - r = await BulletinBoardManager._get_redis() - ids = await r.smembers(f"bb:badges:{user_id}") - await r.close() - return [{"id": bid, **BADGES[bid]} for bid in ids if bid in BADGES] - except Exception: - return [] - - -async def award_badge(user_id: str, badge_id: str) -> bool: - if badge_id not in BADGES: - return False - try: - r = await BulletinBoardManager._get_redis() - ok = await r.sadd(f"bb:badges:{user_id}", badge_id) - await r.close() - return ok > 0 - except Exception: - return False - - -async def get_user_reputation(user_id: str) -> dict: - try: - r = await BulletinBoardManager._get_redis() - p = r.pipeline() - p.get(f"bb:rep:{user_id}") - p.scard(f"bb:badges:{user_id}") - p.get(f"bb:posts:{user_id}") - rep, bc, pc = await p.execute() - await r.close() - return {"reputation": int(rep or 0), "badge_count": bc or 0, "post_count": int(pc or 0)} - except Exception: - return {"reputation": 0, "badge_count": 0, "post_count": 0} - - -X402_BB_POST_PRICE = "$1.00" - - -async def verify_x402_bot(tx_hash: str, bot_addr: str) -> bool: - try: - r = await BulletinBoardManager._get_redis() - if await r.get(f"bb:x402:tx:{tx_hash}"): - await r.close() - return False - await r.setex(f"bb:x402:tx:{tx_hash}", 86400, bot_addr) - await r.setex(f"bb:x402:bot:{bot_addr}", 2592000, "1") # 30 day auth - await r.close() - return True - except Exception: - return False +from app.domains.bulletin import ( # noqa: F401 + BulletinBoardManager, + Comment, + ContentSanitizer, + Post, + PostCategory, + PostStatus, + Priority, + TargetAudience, + award_badge, + get_user_badges, + get_user_reputation, + verify_x402_bot, +) diff --git a/app/core/redis.py b/app/core/redis.py index d2e3e22..19a47cf 100644 --- a/app/core/redis.py +++ b/app/core/redis.py @@ -73,7 +73,7 @@ def get_redis_async() -> "aioredis.Redis | None": return None -def close_redis(): +async def close_redis() -> None: """Close Redis connection pool.""" global _sync_pool, _async_pool if _sync_pool: @@ -85,7 +85,7 @@ def close_redis(): _sync_pool = None if _async_pool: try: - _async_pool.disconnect() + await _async_pool.disconnect() logger.info("Async Redis connection pool closed") except Exception as e: logger.error(f"Async Redis pool close failed: {e}") diff --git a/app/domains/admin/__init__.py b/app/domains/admin/__init__.py new file mode 100644 index 0000000..02af84b --- /dev/null +++ b/app/domains/admin/__init__.py @@ -0,0 +1,31 @@ +"""Admin domain - public API. + +Phase 4 domain consolidation: app.admin_backend -> app.domains.admin. +""" +from __future__ import annotations + +from app.domains.admin.core import ( + PERMISSIONS, + AdminRole, + AdminUserStore, + AuditLogEntry, + AuditLogger, + SecurityManager, + SessionManager, + SystemHealthMonitor, + has_permission, + require_admin, +) + +__all__ = [ + "PERMISSIONS", + "AdminRole", + "AdminUserStore", + "AuditLogEntry", + "AuditLogger", + "SecurityManager", + "SessionManager", + "SystemHealthMonitor", + "has_permission", + "require_admin", +] diff --git a/app/domains/admin/core.py b/app/domains/admin/core.py new file mode 100644 index 0000000..dfb357e --- /dev/null +++ b/app/domains/admin/core.py @@ -0,0 +1,1070 @@ +""" +RMI Admin Backend - Complete Administration System +=================================================== +A hardened, feature-rich admin panel for the RugMunch Intelligence Platform. + +Features: + • Role-Based Access Control (RBAC) - superadmin, admin, moderator, viewer + • Audit Logging - every action logged with IP, timestamp, before/after state + • Rate Limiting - per-endpoint, per-user, per-IP limits + • IP Blocking / Allowlisting - ban bad actors by IP + • 2FA Support - TOTP-based two-factor authentication + • Session Management - active sessions, force logout, expiry control + • System Health - real-time metrics, disk, memory, CPU, service status + • User Management - CRUD, ban/unban, role assignment, activity tracking + • Configuration Management - env vars, feature flags, system settings + • Security Dashboard - failed logins, blocked IPs, threat alerts + • Content Management - announcements, blog posts, SEO settings + • Financial Dashboard - x402 revenue, payment tracking, analytics + • API Key Management - rotate, revoke, scope-limited keys + • Webhook Management - configure, test, monitor webhooks + • Backup & Restore - database, config, snapshot management + +Security: + - All endpoints require admin authentication (JWT + role check) + - Audit log of every admin action (immutable, append-only) + - Rate limiting on all admin endpoints (stricter than public) + - IP allowlist for admin access (optional) + - Session timeout and concurrent session limits + - Password policy enforcement for admin accounts + - Automatic lockout after failed login attempts +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import secrets +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from enum import StrEnum +from typing import Any + +from fastapi import HTTPException, Request + +logger = logging.getLogger("rmi_admin_backend") + + +# ── Role Definitions ────────────────────────────────────────── + + +class AdminRole(StrEnum): + """Admin role hierarchy. Higher = more permissions.""" + + SUPERADMIN = "superadmin" # Full access, can manage other admins + ADMIN = "admin" # Full access except admin management + MODERATOR = "moderator" # Content + user management, no system config + VIEWER = "viewer" # Read-only access to dashboards + SUPPORT = "support" # User management only, read-only system + + +# Permission matrix: role -> set of allowed permissions +PERMISSIONS = { + AdminRole.SUPERADMIN: { + "*", # All permissions + }, + AdminRole.ADMIN: { + "dashboard.read", + "users.read", + "users.write", + "users.ban", + "content.read", + "content.write", + "system.read", + "system.write", + "security.read", + "security.write", + "financial.read", + "financial.write", + "api_keys.read", + "api_keys.write", + "webhooks.read", + "webhooks.write", + "backups.read", + "backups.write", + "settings.read", + "settings.write", + "logs.read", + "analytics.read", + "token_deploy.read", + "token_deploy.write", + }, + AdminRole.MODERATOR: { + "dashboard.read", + "users.read", + "users.write", + "users.ban", + "content.read", + "content.write", + "security.read", + "logs.read", + "analytics.read", + }, + AdminRole.VIEWER: { + "dashboard.read", + "users.read", + "content.read", + "system.read", + "security.read", + "financial.read", + "analytics.read", + "logs.read", + }, + AdminRole.SUPPORT: { + "dashboard.read", + "users.read", + "users.write", + "content.read", + "logs.read", + "analytics.read", + }, +} + + +def has_permission(role: AdminRole, permission: str) -> bool: + """Check if a role has a specific permission.""" + if role == AdminRole.SUPERADMIN: + return True + perms = PERMISSIONS.get(role, set()) + return permission in perms or "*" in perms + + +# ── Audit Log ─────────────────────────────────────────────────── + + +@dataclass +class AuditLogEntry: + """Single audit log entry.""" + + entry_id: str + timestamp: str + admin_id: str + admin_email: str + action: str # e.g., "user.ban", "token.deploy", "config.update" + resource_type: str # e.g., "user", "token", "config" + resource_id: str + ip_address: str + user_agent: str + before_state: dict | None = None + after_state: dict | None = None + status: str = "success" # success, failed, denied + reason: str = "" + session_id: str = "" + request_id: str = "" + + def to_dict(self) -> dict: + return asdict(self) + + +class AuditLogger: + """ + Immutable audit logging system. + Stores to Redis (time-series) + file backup + optional Supabase. + """ + + @staticmethod + async def log( + admin_id: str, + admin_email: str, + action: str, + resource_type: str, + resource_id: str, + ip_address: str = "", + user_agent: str = "", + before_state: dict | None = None, + after_state: dict | None = None, + status: str = "success", + reason: str = "", + session_id: str = "", + request_id: str = "", + ) -> bool: + """Log an admin action.""" + entry = AuditLogEntry( + entry_id=f"audit_{int(time.time() * 1000)}_{secrets.token_hex(4)}", + timestamp=datetime.utcnow().isoformat(), + admin_id=admin_id, + admin_email=admin_email, + action=action, + resource_type=resource_type, + resource_id=resource_id, + ip_address=ip_address, + user_agent=user_agent, + before_state=before_state, + after_state=after_state, + status=status, + reason=reason, + session_id=session_id, + request_id=request_id, + ) + + # Write to Redis (time-series list) + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Add to time-series list + key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" + await r.lpush(key, json.dumps(entry.to_dict())) + + # Keep only last 30 days in Redis + await r.ltrim(key, 0, 9999) + + # Add to admin-specific log + await r.lpush(f"audit_log:admin:{admin_id}", json.dumps(entry.to_dict())) + + # Add to action-specific index + await r.sadd(f"audit_log:actions:{action}", entry.entry_id) + + except Exception as e: + logger.error(f"Redis audit log failed: {e}") + + # Write to local file (append-only, immutable) + try: + log_file = f"/var/log/rmi/audit_{datetime.utcnow().strftime('%Y-%m')}.jsonl" + os.makedirs(os.path.dirname(log_file), exist_ok=True) + with open(log_file, "a") as f: + f.write(json.dumps(entry.to_dict()) + "\n") + except Exception as e: + logger.error(f"File audit log failed: {e}") + + # Write to Supabase if available + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("audit_logs").insert(entry.to_dict()).execute() + except Exception as e: + logger.error(f"Supabase audit log failed: {e}") + + return True + + @staticmethod + async def query( + admin_id: str | None = None, + action: str | None = None, + resource_type: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + limit: int = 100, + offset: int = 0, + ) -> list[AuditLogEntry]: + """Query audit logs.""" + entries = [] + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Get today's log + key = f"audit_log:{datetime.utcnow().strftime('%Y-%m-%d')}" + logs = await r.lrange(key, offset, offset + limit - 1) + + for log in logs: + data = json.loads(log) + + # Filter + if admin_id and data.get("admin_id") != admin_id: + continue + if action and data.get("action") != action: + continue + if resource_type and data.get("resource_type") != resource_type: + continue + + entries.append(AuditLogEntry(**data)) + + except Exception as e: + logger.error(f"Audit query failed: {e}") + + return entries + + +# ── Security Manager ────────────────────────────────────────── + + +class SecurityManager: + """ + Security hardening for admin backend. + - Rate limiting + - IP blocking + - Failed login tracking + - Session management + """ + + # Rate limits: (max_requests, window_seconds) + RATE_LIMITS = { # noqa: RUF012 + "default": (60, 60), # 60/minute + "login": (5, 300), # 5/5min (strict for login) + "admin_write": (30, 60), # 30/minute for write ops + "token_deploy": (10, 3600), # 10/hour for token deploy + "snapshot": (20, 3600), # 20/hour for snapshots + "airdrop": (5, 3600), # 5/hour for airdrops + } + + @staticmethod + async def check_rate_limit( + identifier: str, + limit_type: str = "default", + ) -> dict[str, Any]: + """Check if request is within rate limit.""" + max_req, window = SecurityManager.RATE_LIMITS.get(limit_type, (60, 60)) + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + key = f"rate_limit:{limit_type}:{identifier}" + now = int(time.time()) + + # Clean old entries + await r.zremrangebyscore(key, 0, now - window) + + # Count current requests + count = await r.zcard(key) + + if count >= max_req: + # Get time until reset + oldest = await r.zrange(key, 0, 0, withscores=True) + reset_at = oldest[0][1] + window if oldest else now + window + + return { + "allowed": False, + "remaining": 0, + "reset_at": reset_at, + "limit": max_req, + "window": window, + } + + # Add current request + await r.zadd(key, {str(now): now}) + await r.expire(key, window) + + return { + "allowed": True, + "remaining": max_req - count - 1, + "reset_at": now + window, + "limit": max_req, + "window": window, + } + + except Exception as e: + logger.error(f"Rate limit check failed: {e}") + # Fail open (allow request) if Redis is down + return {"allowed": True, "remaining": -1, "reset_at": 0} + + @staticmethod + async def block_ip(ip: str, reason: str = "", duration_hours: int = 24) -> bool: + """Block an IP address.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + key = f"blocked_ip:{ip}" + data = { + "ip": ip, + "reason": reason, + "blocked_at": datetime.utcnow().isoformat(), + "expires_at": (datetime.utcnow() + timedelta(hours=duration_hours)).isoformat(), + "duration_hours": duration_hours, + } + + await r.setex(key, duration_hours * 3600, json.dumps(data)) + await r.sadd("blocked_ips:all", ip) + + logger.warning(f"IP blocked: {ip} for {duration_hours}h - {reason}") + return True + + except Exception as e: + logger.error(f"IP block failed: {e}") + return False + + @staticmethod + async def unblock_ip(ip: str) -> bool: + """Unblock an IP address.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + await r.delete(f"blocked_ip:{ip}") + await r.srem("blocked_ips:all", ip) + + return True + + except Exception as e: + logger.error(f"IP unblock failed: {e}") + return False + + @staticmethod + async def is_ip_blocked(ip: str) -> dict | None: + """Check if IP is blocked. Returns block info or None.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + data = await r.get(f"blocked_ip:{ip}") + if data: + return json.loads(data) + + return None + + except Exception as e: + logger.error(f"IP block check failed: {e}") + return None + + @staticmethod + async def track_failed_login(identifier: str) -> int: + """Track failed login attempts. Returns current count.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + key = f"failed_logins:{identifier}" + count = await r.incr(key) + await r.expire(key, 3600) # 1 hour window + + # Auto-block after 5 failed attempts + if count >= 5: + ip = identifier.split(":")[-1] if ":" in identifier else identifier + await SecurityManager.block_ip(ip, "Too many failed login attempts", 1) + + return count + + except Exception as e: + logger.error(f"Failed login tracking error: {e}") + return 0 + + @staticmethod + async def reset_failed_login(identifier: str) -> bool: + """Reset failed login counter (on successful login).""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + await r.delete(f"failed_logins:{identifier}") + return True + + except Exception as e: + logger.error(f"Reset failed login error: {e}") + return False + + +# ── Session Manager ─────────────────────────────────────────── + + +class SessionManager: + """ + Admin session management. + - Track active sessions + - Force logout + - Concurrent session limits + - Session expiry + """ + + MAX_CONCURRENT_SESSIONS = 3 + SESSION_TIMEOUT_HOURS = 8 + + @staticmethod + async def create_session( + admin_id: str, + admin_email: str, + role: AdminRole, + ip_address: str, + user_agent: str, + ) -> str: + """Create a new admin session.""" + session_id = f"sess_{secrets.token_urlsafe(16)}" + + session_data = { + "session_id": session_id, + "admin_id": admin_id, + "admin_email": admin_email, + "role": role.value if isinstance(role, AdminRole) else role, + "ip_address": ip_address, + "user_agent": user_agent, + "created_at": datetime.utcnow().isoformat(), + "expires_at": (datetime.utcnow() + timedelta(hours=SessionManager.SESSION_TIMEOUT_HOURS)).isoformat(), + "last_active": datetime.utcnow().isoformat(), + "is_active": True, + } + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Store session + key = f"admin_session:{session_id}" + await r.setex(key, SessionManager.SESSION_TIMEOUT_HOURS * 3600, json.dumps(session_data)) + + # Add to admin's session list + await r.sadd(f"admin_sessions:{admin_id}", session_id) + + # Enforce max concurrent sessions + sessions = await r.smembers(f"admin_sessions:{admin_id}") + if len(sessions) > SessionManager.MAX_CONCURRENT_SESSIONS: + # Remove oldest sessions + sorted_sessions = sorted(sessions) + for old_session in sorted_sessions[: -SessionManager.MAX_CONCURRENT_SESSIONS]: + await SessionManager.destroy_session(old_session) + + return session_id + + except Exception as e: + logger.error(f"Session creation failed: {e}") + return "" + + @staticmethod + async def validate_session(session_id: str) -> dict | None: + """Validate and refresh a session.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + data = await r.get(f"admin_session:{session_id}") + if not data: + return None + + session = json.loads(data) + + # Check expiry + expires = datetime.fromisoformat(session["expires_at"]) + if datetime.utcnow() > expires: + await SessionManager.destroy_session(session_id) + return None + + # Update last active + session["last_active"] = datetime.utcnow().isoformat() + await r.setex( + f"admin_session:{session_id}", + SessionManager.SESSION_TIMEOUT_HOURS * 3600, + json.dumps(session), + ) + + return session + + except Exception as e: + logger.error(f"Session validation failed: {e}") + return None + + @staticmethod + async def destroy_session(session_id: str) -> bool: + """Destroy a session (logout).""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + # Get session data for admin_id + data = await r.get(f"admin_session:{session_id}") + if data: + session = json.loads(data) + admin_id = session.get("admin_id") + if admin_id: + await r.srem(f"admin_sessions:{admin_id}", session_id) + + await r.delete(f"admin_session:{session_id}") + return True + + except Exception as e: + logger.error(f"Session destroy failed: {e}") + return False + + @staticmethod + async def get_active_sessions(admin_id: str) -> list[dict]: + """Get all active sessions for an admin.""" + sessions = [] + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + session_ids = await r.smembers(f"admin_sessions:{admin_id}") + for sid in session_ids: + data = await r.get(f"admin_session:{sid}") + if data: + session = json.loads(data) + # Check if still valid + expires = datetime.fromisoformat(session["expires_at"]) + if datetime.utcnow() < expires: + sessions.append(session) + else: + await r.srem(f"admin_sessions:{admin_id}", sid) + + except Exception as e: + logger.error(f"Get active sessions failed: {e}") + + return sessions + + @staticmethod + async def destroy_all_sessions(admin_id: str) -> int: + """Destroy all sessions for an admin (force logout everywhere).""" + count = 0 + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + session_ids = await r.smembers(f"admin_sessions:{admin_id}") + for sid in session_ids: + await r.delete(f"admin_session:{sid}") + count += 1 + + await r.delete(f"admin_sessions:{admin_id}") + + except Exception as e: + logger.error(f"Destroy all sessions failed: {e}") + + return count + + +# ── Admin User Store ────────────────────────────────────────── + + +class AdminUserStore: + """ + Store and manage admin user accounts. + Uses Redis as primary + Supabase as backup. + """ + + @staticmethod + async def get_admin(admin_id: str) -> dict | None: + """Get admin by ID.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + data = await r.hget("rmi:admins", admin_id) + if data: + return json.loads(data) + + return None + + except Exception as e: + logger.error(f"Get admin failed: {e}") + return None + + @staticmethod + async def get_admin_by_email(email: str) -> dict | None: + """Get admin by email.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + admin_id = await r.hget("rmi:admins:email", email.lower()) + if admin_id: + return await AdminUserStore.get_admin(admin_id) + + return None + + except Exception as e: + logger.error(f"Get admin by email failed: {e}") + return None + + @staticmethod + async def save_admin(admin: dict) -> bool: + """Save admin user.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + await r.hset("rmi:admins", admin["id"], json.dumps(admin)) + await r.hset("rmi:admins:email", admin["email"].lower(), admin["id"]) + + # Also save to Supabase + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("admin_users").upsert(admin).execute() + except Exception: + pass + + return True + + except Exception as e: + logger.error(f"Save admin failed: {e}") + return False + + @staticmethod + async def list_admins() -> list[dict]: + """List all admin users.""" + admins = [] + + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + all_admins = await r.hgetall("rmi:admins") + for data in all_admins.values(): + admins.append(json.loads(data)) + + except Exception as e: + logger.error(f"List admins failed: {e}") + + return admins + + @staticmethod + async def create_admin( + email: str, + password: str, + role: AdminRole = AdminRole.VIEWER, + created_by: str = "", + ) -> dict | None: + """Create a new admin user.""" + from app.domains.auth.passwords import hash_password + + # Check if email already exists + existing = await AdminUserStore.get_admin_by_email(email) + if existing: + return None + + admin_id = hashlib.sha256(email.lower().encode()).hexdigest()[:16] + + admin = { + "id": admin_id, + "email": email, + "password_hash": hash_password(password), + "role": role.value, + "is_active": True, + "created_at": datetime.utcnow().isoformat(), + "created_by": created_by, + "last_login": None, + "login_count": 0, + "two_factor_enabled": False, + "two_factor_secret": None, + "ip_allowlist": [], + "metadata": {}, + } + + await AdminUserStore.save_admin(admin) + + # Remove password hash from returned dict + safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} + return safe_admin + + @staticmethod + async def verify_admin_login(email: str, password: str) -> dict | None: + """Verify admin login credentials.""" + from app.domains.auth.passwords import verify_password + + admin = await AdminUserStore.get_admin_by_email(email) + if not admin: + return None + + if not admin.get("is_active", True): + return None + + if not verify_password(password, admin.get("password_hash", "")): + return None + + # Update last login + admin["last_login"] = datetime.utcnow().isoformat() + admin["login_count"] = admin.get("login_count", 0) + 1 + await AdminUserStore.save_admin(admin) + + # Remove sensitive data + safe_admin = {k: v for k, v in admin.items() if k != "password_hash" and k != "two_factor_secret"} + return safe_admin + + +# ── System Health Monitor ───────────────────────────────────── + + +class SystemHealthMonitor: + """ + Monitor system health and performance. + """ + + @staticmethod + async def get_system_health() -> dict[str, Any]: + """Get comprehensive system health.""" + import psutil + + # CPU + cpu_percent = psutil.cpu_percent(interval=0.5) + cpu_count = psutil.cpu_count() + + # Memory + memory = psutil.virtual_memory() + + # Disk + disk = psutil.disk_usage("/") + + # Network + net_io = psutil.net_io_counters() + + # Process info + backend_pid = os.getpid() + backend_proc = psutil.Process(backend_pid) + + # Check services + services = { + "redis": await SystemHealthMonitor._check_redis(), + "supabase": await SystemHealthMonitor._check_supabase(), + "backend": {"status": "running", "pid": backend_pid}, + } + + return { + "timestamp": datetime.utcnow().isoformat(), + "cpu": { + "percent": cpu_percent, + "count": cpu_count, + "per_cpu": psutil.cpu_percent(percpu=True, interval=0.1), + }, + "memory": { + "total_gb": round(memory.total / (1024**3), 2), + "available_gb": round(memory.available / (1024**3), 2), + "percent": memory.percent, + "used_gb": round(memory.used / (1024**3), 2), + }, + "disk": { + "total_gb": round(disk.total / (1024**3), 2), + "used_gb": round(disk.used / (1024**3), 2), + "free_gb": round(disk.free / (1024**3), 2), + "percent": round(disk.used / disk.total * 100, 1), + }, + "network": { + "bytes_sent_mb": round(net_io.bytes_sent / (1024**2), 2), + "bytes_recv_mb": round(net_io.bytes_recv / (1024**2), 2), + }, + "backend": { + "pid": backend_pid, + "memory_mb": round(backend_proc.memory_info().rss / (1024**2), 2), + "cpu_percent": backend_proc.cpu_percent(interval=0.2), + "threads": backend_proc.num_threads(), + "open_files": len(backend_proc.open_files()), + "connections": len(backend_proc.connections()), + }, + "services": services, + "uptime_seconds": int(time.time() - psutil.boot_time()), + } + + @staticmethod + async def _check_redis() -> dict: + """Check Redis connection.""" + try: + import redis.asyncio as redis_lib + + r = redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + await r.ping() + info = await r.info() + return { + "status": "connected", + "version": info.get("redis_version", "unknown"), + "used_memory_mb": round(info.get("used_memory", 0) / (1024**2), 2), + "connected_clients": info.get("connected_clients", 0), + "total_keys": info.get("db0", {}).get("keys", 0) if isinstance(info.get("db0"), dict) else 0, + } + except Exception as e: + return {"status": "error", "error": str(e)} + + @staticmethod + async def _check_supabase() -> dict: + """Check Supabase connection.""" + try: + supabase_url = os.getenv("SUPABASE_URL") + if not supabase_url: + return {"status": "not_configured"} + + import httpx + + async with httpx.AsyncClient() as client: + r = await client.get(f"{supabase_url}/rest/v1/", timeout=5) + return { + "status": "connected" if r.status_code < 500 else "error", + "url": supabase_url, + "response_code": r.status_code, + } + except Exception as e: + return {"status": "error", "error": str(e)} + + +# ── Convenience: Admin Auth Decorator ───────────────────────── + + +async def require_admin( + request: Request, + required_permission: str = "", + min_role: AdminRole = AdminRole.VIEWER, +) -> dict[str, Any]: + """ + Verify admin authentication and authorization. + + Usage: + admin = await require_admin(request, "users.write", AdminRole.ADMIN) + """ + # Get session token from header + session_token = request.headers.get("X-Admin-Session", "") + if not session_token: + raise HTTPException(status_code=401, detail="Admin session required") + + # Validate session + session = await SessionManager.validate_session(session_token) + if not session: + raise HTTPException(status_code=401, detail="Invalid or expired session") + + # Check IP block + client_ip = request.client.host if request.client else "" + block_info = await SecurityManager.is_ip_blocked(client_ip) + if block_info: + raise HTTPException(status_code=403, detail="IP blocked") + + # Get admin + admin = await AdminUserStore.get_admin(session["admin_id"]) + if not admin or not admin.get("is_active", True): + raise HTTPException(status_code=403, detail="Admin account inactive") + + # Check role level + role = AdminRole(admin.get("role", "viewer")) + role_order = { + AdminRole.VIEWER: 0, + AdminRole.SUPPORT: 1, + AdminRole.MODERATOR: 2, + AdminRole.ADMIN: 3, + AdminRole.SUPERADMIN: 4, + } + if role_order[role] < role_order[min_role]: + raise HTTPException(status_code=403, detail="Insufficient privileges") + + # Check specific permission + if required_permission and not has_permission(role, required_permission): + raise HTTPException(status_code=403, detail="Permission denied") + + # Check rate limit + rate_check = await SecurityManager.check_rate_limit( + f"admin:{admin['id']}", + "admin_write" if required_permission.endswith(".write") else "default", + ) + if not rate_check["allowed"]: + raise HTTPException(status_code=429, detail=f"Rate limit exceeded. Reset at {rate_check['reset_at']}") + + # Log access + await AuditLogger.log( + admin_id=admin["id"], + admin_email=admin["email"], + action="admin.access", + resource_type="endpoint", + resource_id=str(request.url.path), + ip_address=client_ip, + user_agent=request.headers.get("user-agent", ""), + session_id=session_token, + ) + + return { + "admin": admin, + "session": session, + "role": role, + } diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index 4479e4d..deb69f7 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -5,6 +5,7 @@ Auth Router - Complete authentication system (email, wallet, OAuth, Telegram) import json import logging from datetime import datetime, timedelta +from typing import Any, cast from fastapi import APIRouter, HTTPException, Request @@ -77,7 +78,7 @@ router.include_router(oauth_router, tags=["auth"]) # ── Storage (Redis-backed user store) ── # ── Email Auth ── @router.post("/register", response_model=WalletAuthResponse) -def register_email(req: EmailRegisterRequest): +def register_email(req: EmailRegisterRequest) -> WalletAuthResponse: """Register a new user with email/password.""" if not _is_valid_email(req.email): raise HTTPException(status_code=400, detail="Invalid email format") @@ -93,7 +94,7 @@ def register_email(req: EmailRegisterRequest): user_id = _derive_user_id(req.email) hashed = hash_password(req.password) - user = { + user: dict[str, Any] = { "id": user_id, "email": req.email, "display_name": req.display_name, @@ -112,29 +113,33 @@ def register_email(req: EmailRegisterRequest): # Save user + email index r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", req.email.lower(), user_id) token = _create_jwt(user_id, req.email, "FREE", "USER", req.wallet_address) - return { - "access_token": token, - "refresh_token": token, - "user": { - "id": user_id, - "email": req.email, - "display_name": req.display_name, - "wallet_address": req.wallet_address, - "wallet_chain": req.wallet_chain, - "tier": "FREE", - "role": "USER", - "created_at": user["created_at"], + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user_id, + "email": req.email, + "display_name": req.display_name, + "wallet_address": req.wallet_address, + "wallet_chain": req.wallet_chain, + "tier": "FREE", + "role": "USER", + "created_at": user["created_at"], + }, }, - } + ) @router.post("/login", response_model=WalletAuthResponse) -def login_email(req: EmailLoginRequest): +def login_email(req: EmailLoginRequest) -> WalletAuthResponse: """Login with email/password. If 2FA enabled, returns partial token requiring /2fa/login.""" user = _get_user_by_email(req.email) if not user or not user.get("password_hash"): @@ -162,21 +167,24 @@ def login_email(req: EmailLoginRequest): payload["pre_auth"] = True pre_token = jose_jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) - return { - "access_token": pre_token, - "refresh_token": pre_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - "_2fa_required": True, + return cast( + "WalletAuthResponse", + { + "access_token": pre_token, + "refresh_token": pre_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + "_2fa_required": True, + }, }, - } + ) token = _create_jwt( user["id"], @@ -186,38 +194,37 @@ def login_email(req: EmailLoginRequest): user.get("wallet_address"), ) - return { - "access_token": token, - "refresh_token": token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) # ── Wallet Auth ── @router.post("/wallet/nonce", response_model=NonceResponse) -async def wallet_nonce(req: WalletNonceRequest): +async def wallet_nonce(req: WalletNonceRequest) -> NonceResponse: """Get a nonce for wallet signature.""" nonce = generate_nonce() timestamp = datetime.utcnow().isoformat() message = f"RugMunch Intelligence wants you to sign in with your {req.chain.title()} account.\n\nWallet: {req.address}\nNonce: {nonce}\nTimestamp: {timestamp}" - return { - "nonce": nonce, - "timestamp": timestamp, - "message": message, - } + return cast("NonceResponse", {"nonce": nonce, "timestamp": timestamp, "message": message}) @router.post("/wallet/verify", response_model=WalletAuthResponse) -async def wallet_verify(req: WalletVerifyRequest): +async def wallet_verify(req: WalletVerifyRequest) -> WalletAuthResponse: """Verify wallet signature and create session.""" from app.domains.auth.wallet import get_or_create_wallet_user, verify_wallet_signature @@ -226,25 +233,28 @@ async def wallet_verify(req: WalletVerifyRequest): user_data = await get_or_create_wallet_user(req.address) - return { - "access_token": user_data["access_token"], - "refresh_token": user_data["refresh_token"], - "user": { - "id": user_data["id"], - "email": user_data["email"], - "display_name": user_data.get("display_name", f"Agent {req.address[2:8].upper()}"), - "wallet_address": req.address, - "wallet_chain": req.chain, - "tier": user_data["tier"], - "role": user_data["role"], - "created_at": user_data["created_at"], + return cast( + "WalletAuthResponse", + { + "access_token": user_data["access_token"], + "refresh_token": user_data["refresh_token"], + "user": { + "id": user_data["id"], + "email": user_data["email"], + "display_name": user_data.get("display_name", f"Agent {req.address[2:8].upper()}"), + "wallet_address": req.address, + "wallet_chain": req.chain, + "tier": user_data["tier"], + "role": user_data["role"], + "created_at": user_data["created_at"], + }, }, - } + ) # ── User Profile ── @router.get("/user/me", response_model=UserResponse) -async def user_me(request: Request): +async def user_me(request: Request) -> UserResponse: """Get current authenticated user profile.""" auth_header = request.headers.get("Authorization", "") if not auth_header.startswith("Bearer "): @@ -259,26 +269,29 @@ async def user_me(request: Request): if not user: raise HTTPException(status_code=404, detail="User not found") - return { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - "xp": user.get("xp", 0), - "level": user.get("level", 1), - "badges": user.get("badges", []), - } + return cast( + "UserResponse", + { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + "xp": user.get("xp", 0), + "level": user.get("level", 1), + "badges": user.get("badges", []), + }, + ) # ── 2FA / TOTP Endpoints ── @router.post("/2fa/setup") -async def twofa_setup(request: Request): +async def twofa_setup(request: Request) -> TwoFASetupResponse: """Begin 2FA setup - generate secret + QR code. Returns plaintext secret + backup codes (shown once).""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -300,6 +313,7 @@ async def twofa_setup(request: Request): # Store pending secret (not enabled until verified) r = get_redis() + assert r is not None pending = { "secret": secret, "backup_codes": [_hash_backup_code(c) for c in backup_codes], @@ -307,16 +321,16 @@ async def twofa_setup(request: Request): } r.setex(f"rmi:2fa_pending:{user['id']}", timedelta(minutes=10), json.dumps(pending)) - return { + return cast("TwoFASetupResponse", { "secret": secret, "qr_code": qr_b64, "uri": uri, "backup_codes": backup_codes, # plaintext - shown once - } + }) @router.post("/2fa/enable") -async def twofa_enable(req: TwoFAEnableRequest, request: Request): +async def twofa_enable(req: TwoFAEnableRequest, request: Request) -> dict[str, Any]: """Enable 2FA - verify the first TOTP code, then persist.""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -326,6 +340,7 @@ async def twofa_enable(req: TwoFAEnableRequest, request: Request): raise HTTPException(status_code=401, detail="Authentication required") r = get_redis() + assert r is not None pending_raw = r.get(f"rmi:2fa_pending:{user['id']}") if not pending_raw: raise HTTPException(status_code=400, detail="No pending 2FA setup. Call /2fa/setup first.") @@ -357,7 +372,7 @@ async def twofa_enable(req: TwoFAEnableRequest, request: Request): @router.post("/2fa/disable") -async def twofa_disable(request: Request): +async def twofa_disable(request: Request) -> dict[str, Any]: """Disable 2FA - requires current password for security.""" user = await get_current_user(request) if not user: @@ -379,27 +394,27 @@ async def twofa_disable(request: Request): _save_user(user) logger.info(f"[2FA DISABLED] user={user['id']} email={user['email']}") - return {"status": "ok", "message": "2FA disabled successfully"} + return cast("dict[str, Any]", {"status": "ok", "message": "2FA disabled successfully"}) @router.get("/2fa/status", response_model=TwoFAStatusResponse) -async def twofa_status(request: Request): +async def twofa_status(request: Request) -> TwoFAStatusResponse: """Check 2FA status for current user.""" user = await get_current_user(request) if not user: raise HTTPException(status_code=401, detail="Authentication required") enabled = user.get("totp_enabled", False) - return { + return cast("TwoFAStatusResponse", { "enabled": enabled, "method": "totp" if enabled else "none", "created_at": user.get("totp_created_at"), "last_used": user.get("totp_last_used"), - } + }) @router.post("/2fa/verify") -async def twofa_verify(req: TwoFAVerifyRequest, request: Request): +async def twofa_verify(req: TwoFAVerifyRequest, request: Request) -> dict[str, Any]: """Verify a TOTP code (for re-auth during sensitive operations).""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -418,11 +433,11 @@ async def twofa_verify(req: TwoFAVerifyRequest, request: Request): user["totp_last_used"] = datetime.utcnow().isoformat() _save_user(user) - return {"status": "ok", "verified": True} + return cast("dict[str, Any]", {"status": "ok", "verified": True}) @router.post("/2fa/login") -async def twofa_login(req: TwoFALoginRequest): +async def twofa_login(req: TwoFALoginRequest) -> WalletAuthResponse: """Login with email + password + 2FA code. Returns full JWT on success.""" if not PYOTP_AVAILABLE: raise HTTPException(status_code=503, detail="2FA service unavailable") @@ -473,17 +488,20 @@ async def twofa_login(req: TwoFALoginRequest): logger.info(f"[2FA LOGIN] user={user['id']} email={user['email']}") - return { - "access_token": token, - "refresh_token": token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) diff --git a/app/domains/auth/jwt.py b/app/domains/auth/jwt.py index 6d63428..b776b58 100644 --- a/app/domains/auth/jwt.py +++ b/app/domains/auth/jwt.py @@ -41,7 +41,7 @@ def _create_jwt( } if wallet: payload["wallet"] = wallet - return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) + return str(jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM)) def _verify_jwt(token: str) -> dict[str, Any] | None: diff --git a/app/domains/auth/oauth.py b/app/domains/auth/oauth.py index c2f4c27..812ed9b 100644 --- a/app/domains/auth/oauth.py +++ b/app/domains/auth/oauth.py @@ -7,8 +7,10 @@ from __future__ import annotations import json import os from datetime import datetime +from typing import cast from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import RedirectResponse from app.core.redis import get_redis from app.domains.auth.jwt import _create_jwt, _derive_user_id @@ -25,7 +27,7 @@ FRONTEND_URL = os.getenv("FRONTEND_URL", "https://rugmunch.io") @router.get("/google/url", response_model=GoogleAuthResponse) -async def google_auth_url(): +async def google_auth_url() -> GoogleAuthResponse: """Get Google OAuth URL.""" client_id = os.getenv("GOOGLE_CLIENT_ID") redirect_uri = os.getenv( @@ -33,7 +35,7 @@ async def google_auth_url(): ) if not client_id: - return {"url": "/auth/callback?error=google_not_configured"} + return cast("GoogleAuthResponse", {"url": "/auth/callback?error=google_not_configured"}) from urllib.parse import urlencode @@ -46,14 +48,12 @@ async def google_auth_url(): "prompt": "consent", } url = f"https://accounts.google.com/o/oauth2/v2/auth?{urlencode(params)}" - return {"url": url} + return cast("GoogleAuthResponse", {"url": url}) @router.get("/google/callback") -async def google_callback(request: Request): +async def google_callback(request: Request) -> RedirectResponse: """Handle Google OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - code = request.query_params.get("code") error = request.query_params.get("error") @@ -118,6 +118,7 @@ async def google_callback(request: Request): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) @@ -129,7 +130,7 @@ async def google_callback(request: Request): @router.post("/google/callback", response_model=WalletAuthResponse) -async def google_callback_post(request: Request): +async def google_callback_post(request: Request) -> WalletAuthResponse: """Legacy POST endpoint for manual code exchange.""" body = await request.json() code = body.get("code") @@ -192,6 +193,7 @@ async def google_callback_post(request: Request): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) @@ -199,24 +201,27 @@ async def google_callback_post(request: Request): user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") ) - return { - "access_token": jwt_token, - "refresh_token": jwt_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", email), - "wallet_address": None, - "wallet_chain": None, - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": jwt_token, + "refresh_token": jwt_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", email), + "wallet_address": None, + "wallet_chain": None, + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) @router.post("/telegram", response_model=WalletAuthResponse) -async def telegram_auth(req: TelegramAuthRequest): +async def telegram_auth(req: TelegramAuthRequest) -> WalletAuthResponse: """Authenticate via Telegram Web App data.""" bot_token = os.getenv("TELEGRAM_BOT_TOKEN") if not bot_token: @@ -272,6 +277,7 @@ async def telegram_auth(req: TelegramAuthRequest): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", telegram_user_id, json.dumps(user)) r.hset("rmi:users:telegram", str(req.id), telegram_user_id) @@ -279,24 +285,27 @@ async def telegram_auth(req: TelegramAuthRequest): user["id"], user["email"], user.get("tier", "FREE"), user.get("role", "USER") ) - return { - "access_token": jwt_token, - "refresh_token": jwt_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name"), - "wallet_address": None, - "wallet_chain": None, - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), + return cast( + "WalletAuthResponse", + { + "access_token": jwt_token, + "refresh_token": jwt_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name"), + "wallet_address": None, + "wallet_chain": None, + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, }, - } + ) @router.get("/github/url", response_model=GoogleAuthResponse) -async def github_auth_url(): +async def github_auth_url() -> GoogleAuthResponse: """Get GitHub OAuth URL.""" client_id = os.getenv("GITHUB_CLIENT_ID") redirect_uri = os.getenv( @@ -304,7 +313,7 @@ async def github_auth_url(): ) if not client_id: - return {"url": "/auth/callback?error=github_not_configured"} + return cast("GoogleAuthResponse", {"url": "/auth/callback?error=github_not_configured"}) from urllib.parse import urlencode @@ -314,14 +323,12 @@ async def github_auth_url(): "scope": "user:email", } url = f"https://github.com/login/oauth/authorize?{urlencode(params)}" - return {"url": url} + return cast("GoogleAuthResponse", {"url": url}) @router.get("/github/callback") -async def github_callback(request: Request): +async def github_callback(request: Request) -> RedirectResponse: """Handle GitHub OAuth redirect - exchanges code, creates user, redirects to frontend with token.""" - from fastapi.responses import RedirectResponse - code = request.query_params.get("code") error = request.query_params.get("error") @@ -403,6 +410,7 @@ async def github_callback(request: Request): "scans_used": 0, } r = get_redis() + assert r is not None r.hset("rmi:users", user_id, json.dumps(user)) r.hset("rmi:users:email", email.lower(), user_id) diff --git a/app/domains/auth/schemas.py b/app/domains/auth/schemas.py index 5a964f1..dce6f80 100644 --- a/app/domains/auth/schemas.py +++ b/app/domains/auth/schemas.py @@ -58,7 +58,7 @@ class UserResponse(BaseModel): created_at: str xp: int = 0 level: int = 1 - badges: list = [] + badges: list[str] = [] class GoogleAuthResponse(BaseModel): @@ -80,7 +80,7 @@ class TwoFASetupResponse(BaseModel): secret: str qr_code: str # base64 data URI uri: str # otpauth:// URI - backup_codes: list # plaintext - shown once + backup_codes: list[str] # plaintext - shown once class TwoFAEnableRequest(BaseModel): diff --git a/app/domains/auth/store.py b/app/domains/auth/store.py index 63be999..f398002 100644 --- a/app/domains/auth/store.py +++ b/app/domains/auth/store.py @@ -10,7 +10,7 @@ from __future__ import annotations import json import re -from typing import Any +from typing import Any, cast from app.core.redis import get_redis @@ -24,7 +24,7 @@ def _get_user(user_id: str) -> dict[str, Any] | None: if not raw: return None try: - return json.loads(raw) + return cast("dict[str, Any]", json.loads(raw)) except (json.JSONDecodeError, TypeError): return None diff --git a/app/domains/auth/totp.py b/app/domains/auth/totp.py index a84c95f..ff30494 100644 --- a/app/domains/auth/totp.py +++ b/app/domains/auth/totp.py @@ -41,7 +41,7 @@ def _generate_totp_secret() -> str: """Generate a new base32 TOTP secret.""" if not PYOTP_AVAILABLE: raise RuntimeError("pyotp not installed") - return pyotp.random_base32() + return str(pyotp.random_base32()) def _build_totp(secret: str) -> Any: @@ -54,13 +54,13 @@ def _verify_totp(secret: str, code: str) -> bool: if not PYOTP_AVAILABLE or not secret or not code: return False totp = _build_totp(secret) - return totp.verify(code, valid_window=TOTP_WINDOW) + return bool(totp.verify(code, valid_window=TOTP_WINDOW)) def _get_totp_uri(secret: str, email: str) -> str: """Get otpauth:// URI for QR code generation.""" totp = _build_totp(secret) - return totp.provisioning_uri(name=email, issuer_name=TOTP_ISSUER) + return str(totp.provisioning_uri(name=email, issuer_name=TOTP_ISSUER)) def _generate_qr_base64(uri: str) -> str: @@ -71,7 +71,7 @@ def _generate_qr_base64(uri: str) -> str: return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() -def _generate_backup_codes(count: int = 8) -> list: +def _generate_backup_codes(count: int = 8) -> list[str]: """Generate single-use backup codes.""" codes = [] for _ in range(count): diff --git a/app/domains/auth/wallet.py b/app/domains/auth/wallet.py index b4b222b..ebfeb76 100644 --- a/app/domains/auth/wallet.py +++ b/app/domains/auth/wallet.py @@ -33,7 +33,7 @@ def verify_wallet_signature(message: str, signature: str, address: str) -> bool: return len(signature) >= 60 -def decode_signature(signature: str) -> tuple: +def decode_signature(signature: str) -> tuple[str, str, int]: """ Decode a wallet signature into r, s, v components. diff --git a/app/domains/billing/x402/tools/integration.py b/app/domains/billing/x402/tools/integration.py index 59b1689..05bb131 100644 --- a/app/domains/billing/x402/tools/integration.py +++ b/app/domains/billing/x402/tools/integration.py @@ -388,7 +388,7 @@ async def framework_discovery(): "free": True, }, "mcp": { - "endpoint": "python -m app.mcp.x402_mcp_server", + "endpoint": "python -m app.domains.mcp.x402_mcp_server", "format": "Model Context Protocol (stdio)", "usage": "Claude Desktop, Claude Code, any MCP client", "free": True, diff --git a/app/domains/bulletin/__init__.py b/app/domains/bulletin/__init__.py new file mode 100644 index 0000000..d4a4b72 --- /dev/null +++ b/app/domains/bulletin/__init__.py @@ -0,0 +1,35 @@ +"""Bulletin domain - public API. + +Phase 4 domain consolidation: app.bulletin_board -> app.domains.bulletin. +""" +from __future__ import annotations + +from app.domains.bulletin.core import ( + BulletinBoardManager, + Comment, + ContentSanitizer, + Post, + PostCategory, + PostStatus, + Priority, + TargetAudience, + award_badge, + get_user_badges, + get_user_reputation, + verify_x402_bot, +) + +__all__ = [ + "BulletinBoardManager", + "Comment", + "ContentSanitizer", + "Post", + "PostCategory", + "PostStatus", + "Priority", + "TargetAudience", + "award_badge", + "get_user_badges", + "get_user_reputation", + "verify_x402_bot", +] diff --git a/app/domains/bulletin/core.py b/app/domains/bulletin/core.py new file mode 100644 index 0000000..3948113 --- /dev/null +++ b/app/domains/bulletin/core.py @@ -0,0 +1,902 @@ +""" +RMI Bulletin Board Management System +===================================== +A full content management backend for announcements, news, alerts, platform +communications, and community bulletin boards. + +Features: + • Posts - CRUD with rich text, attachments, scheduling, expiry + • Categories - organize by type (news, alert, update, promo, system) + • Targeting - audience segmentation (all, free, premium, admins, specific tiers) + • Moderation - draft/review/published/archived workflow, approval chains + • Pinning - sticky posts, priority ordering + • Analytics - views, clicks, engagement tracking per post + • Comments - threaded discussions on posts (optional) + • Notifications - push/email/Telegram alerts for critical posts + • Scheduling - publish at future date, auto-archive after expiry + • Versioning - track edit history, rollback capability + • Search - full-text search across all posts + • SEO - slug generation, meta tags, OpenGraph + +Security: + - All write operations require admin auth + content.write permission + - Audit log of every content change + - Rate limiting on publish operations + - Content sanitization (strip XSS, validate HTML) +""" + +from __future__ import annotations + +import html +import json +import logging +import os +import re +import time +from dataclasses import asdict, dataclass, field +from datetime import datetime +from enum import StrEnum +from typing import Any, ClassVar + +logger = logging.getLogger("rmi_bulletin_board") + + +# ── Enums ───────────────────────────────────────────────────── + + +class PostStatus(StrEnum): + DRAFT = "draft" + REVIEW = "review" + PUBLISHED = "published" + ARCHIVED = "archived" + SCHEDULED = "scheduled" + + +class PostCategory(StrEnum): + NEWS = "news" # Platform news + ALERT = "alert" # Security/urgent alerts + UPDATE = "update" # Feature updates + PROMO = "promo" # Promotions/offers + SYSTEM = "system" # System maintenance + COMMUNITY = "community" # Community posts + ANNOUNCEMENT = "announcement" # General announcements + TUTORIAL = "tutorial" # Guides/how-tos + + +class TargetAudience(StrEnum): + ALL = "all" + FREE = "free" + PREMIUM = "premium" + PRO = "pro" + ENTERPRISE = "enterprise" + ADMINS = "admins" + MODERATORS = "moderators" + + +class Priority(StrEnum): + LOW = "low" + NORMAL = "normal" + HIGH = "high" + CRITICAL = "critical" + + +# ── Data Models ───────────────────────────────────────────── + + +@dataclass +class Post: + """Bulletin board post.""" + + post_id: str + title: str + slug: str + content: str + summary: str + category: str + status: str + priority: str + target_audience: str + author_id: str + author_email: str + author_name: str + created_at: str + updated_at: str + published_at: str | None = None + scheduled_at: str | None = None + expires_at: str | None = None + archived_at: str | None = None + pinned: bool = False + pin_order: int = 0 + featured_image: str = "" + attachments: list[dict] = field(default_factory=list) + tags: list[str] = field(default_factory=list) + meta_title: str = "" + meta_description: str = "" + og_image: str = "" + view_count: int = 0 + click_count: int = 0 + engagement_score: float = 0.0 + version: int = 1 + edit_history: list[dict] = field(default_factory=list) + approved_by: str = "" + approved_at: str | None = None + notification_sent: bool = False + allow_comments: bool = False + comments_count: int = 0 + + def to_dict(self) -> dict: + return asdict(self) + + def to_public_dict(self) -> dict: + """Return public-safe version (no internal fields).""" + return { + "post_id": self.post_id, + "title": self.title, + "slug": self.slug, + "content": self.content, + "summary": self.summary, + "category": self.category, + "priority": self.priority, + "author_name": self.author_name, + "created_at": self.created_at, + "published_at": self.published_at, + "expires_at": self.expires_at, + "pinned": self.pinned, + "pin_order": self.pin_order, + "featured_image": self.featured_image, + "attachments": self.attachments, + "tags": self.tags, + "meta_title": self.meta_title, + "meta_description": self.meta_description, + "og_image": self.og_image, + "view_count": self.view_count, + "allow_comments": self.allow_comments, + "comments_count": self.comments_count, + } + + +@dataclass +class Comment: + """Comment on a post.""" + + comment_id: str + post_id: str + author_id: str + author_name: str + author_email: str + content: str + created_at: str + updated_at: str + parent_id: str | None = None + status: str = "approved" # approved, pending, rejected + likes: int = 0 + replies: list[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return asdict(self) + + +# ── Content Sanitizer ─────────────────────────────────────── + + +class ContentSanitizer: + """Sanitize user-generated content to prevent XSS.""" + + ALLOWED_TAGS: ClassVar[dict] ={ + "p", + "br", + "strong", + "b", + "em", + "i", + "u", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "ul", + "ol", + "li", + "a", + "img", + "blockquote", + "code", + "pre", + "table", + "thead", + "tbody", + "tr", + "td", + "th", + "div", + "span", + "hr", + "sub", + "sup", + "del", + "ins", + } + + ALLOWED_ATTRS: ClassVar[dict] ={ + "a": ["href", "title", "target"], + "img": ["src", "alt", "title", "width", "height"], + "div": ["class"], + "span": ["class"], + "code": ["class"], + "pre": ["class"], + } + + @staticmethod + def sanitize(text: str) -> str: + """Basic HTML sanitization.""" + if not text: + return "" + + # Escape HTML entities first + text = html.escape(text) + + # Then selectively un-escape allowed tags + # This is a simplified approach - in production use bleach or similar + # For now, strip all HTML tags for safety + text = re.sub(r"<[^>]+>", "", text) + + return text.strip() + + @staticmethod + def generate_slug(title: str) -> str: + """Generate URL-friendly slug from title.""" + slug = re.sub(r"[^\w\s-]", "", title.lower()) + slug = re.sub(r"[-\s]+", "-", slug) + return slug[:80] + + @staticmethod + def generate_summary(content: str, max_length: int = 200) -> str: + """Generate summary from content.""" + # Strip HTML + text = re.sub(r"<[^>]+>", "", content) + text = text.replace("\n", " ").strip() + if len(text) > max_length: + text = text[:max_length].rsplit(" ", 1)[0] + "..." + return text + + +# ── Bulletin Board Manager ──────────────────────────────────── + + +class BulletinBoardManager: + """ + Core manager for bulletin board operations. + Uses Redis as primary store + Supabase backup. + """ + + @staticmethod + async def _get_redis(): + import redis.asyncio as redis_lib + + return redis_lib.Redis( + host=os.getenv("REDIS_HOST", "localhost"), + port=int(os.getenv("REDIS_PORT", "6379")), + password=os.getenv("REDIS_PASSWORD", ""), + decode_responses=True, + ) + + @staticmethod + async def create_post( + title: str, + content: str, + category: str, + author_id: str, + author_email: str, + author_name: str = "", + priority: str = "normal", + target_audience: str = "all", + status: str = "draft", + featured_image: str = "", + attachments: list[dict] | None = None, + tags: list[str] | None = None, + scheduled_at: str | None = None, + expires_at: str | None = None, + pinned: bool = False, + allow_comments: bool = False, + meta_title: str = "", + meta_description: str = "", + ) -> Post: + """Create a new post.""" + post_id = f"post_{int(time.time())}_{os.urandom(4).hex()}" + slug = ContentSanitizer.generate_slug(title) + summary = ContentSanitizer.generate_summary(content) + now = datetime.utcnow().isoformat() + + # Sanitize content + content = ContentSanitizer.sanitize(content) + + post = Post( + post_id=post_id, + title=title[:200], + slug=slug, + content=content, + summary=summary, + category=category, + status=status, + priority=priority, + target_audience=target_audience, + author_id=author_id, + author_email=author_email, + author_name=author_name or author_email.split("@")[0], + created_at=now, + updated_at=now, + scheduled_at=scheduled_at, + expires_at=expires_at, + pinned=pinned, + featured_image=featured_image, + attachments=attachments or [], + tags=tags or [], + meta_title=meta_title or title[:70], + meta_description=meta_description or summary[:160], + allow_comments=allow_comments, + ) + + r = await BulletinBoardManager._get_redis() + + # Save post + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post.to_dict())) + + # Add to category index + await r.sadd(f"bulletin:category:{category}", post_id) + + # Add to status index + await r.sadd(f"bulletin:status:{status}", post_id) + + # Add to author index + await r.sadd(f"bulletin:author:{author_id}", post_id) + + # Add to audience index + await r.sadd(f"bulletin:audience:{target_audience}", post_id) + + # Add to pinned index if pinned + if pinned: + await r.sadd("bulletin:pinned", post_id) + await r.zadd("bulletin:pinned_order", {post_id: post.pin_order}) + + # Add to scheduled index if scheduled + if scheduled_at and status == "scheduled": + ts = int(datetime.fromisoformat(scheduled_at).timestamp()) + await r.zadd("bulletin:scheduled", {post_id: ts}) + + # Add to search index (simple word index) + words = set(re.findall(r"\w+", title.lower() + " " + content.lower())) + for word in words: + if len(word) > 2: + await r.sadd(f"bulletin:search:{word}", post_id) + + # Save to Supabase + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("bulletin_posts").insert(post.to_dict()).execute() + except Exception as e: + logger.error(f"Supabase bulletin post save failed: {e}") + + return post + + @staticmethod + async def get_post(post_id: str, increment_views: bool = False) -> Post | None: + """Get a post by ID.""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return None + + post_dict = json.loads(data) + + if increment_views and post_dict.get("status") == "published": + post_dict["view_count"] = post_dict.get("view_count", 0) + 1 + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + + return Post(**post_dict) + + @staticmethod + async def get_post_by_slug(slug: str) -> Post | None: + """Get a post by slug.""" + r = await BulletinBoardManager._get_redis() + # Search all posts for matching slug + all_posts = await r.hgetall("rmi:bulletin_posts") + for _post_id, data in all_posts.items(): + post_dict = json.loads(data) + if post_dict.get("slug") == slug: + return Post(**post_dict) + return None + + @staticmethod + async def update_post( + post_id: str, + updates: dict[str, Any], + editor_id: str = "", + editor_email: str = "", + ) -> Post | None: + """Update a post with versioning.""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return None + + post_dict = json.loads(data) + before_state = {k: post_dict.get(k) for k in updates if k in post_dict} + + # Record edit history + edit_entry = { + "edited_at": datetime.utcnow().isoformat(), + "edited_by": editor_id, + "editor_email": editor_email, + "changes": before_state, + "version": post_dict.get("version", 1), + } + + history = post_dict.get("edit_history", []) + history.append(edit_entry) + post_dict["edit_history"] = history + post_dict["version"] = post_dict.get("version", 1) + 1 + post_dict["updated_at"] = datetime.utcnow().isoformat() + + # Apply updates + for key, value in updates.items(): + if key == "content": + value = ContentSanitizer.sanitize(value) + post_dict["summary"] = ContentSanitizer.generate_summary(value) + if key == "title": + post_dict["slug"] = ContentSanitizer.generate_slug(value) + post_dict[key] = value + + # Handle status transitions + old_status = before_state.get("status") + new_status = post_dict.get("status") + if old_status != new_status: + # Update status indexes + if old_status: + await r.srem(f"bulletin:status:{old_status}", post_id) + await r.sadd(f"bulletin:status:{new_status}", post_id) + + if new_status == "published": + post_dict["published_at"] = datetime.utcnow().isoformat() + elif new_status == "archived": + post_dict["archived_at"] = datetime.utcnow().isoformat() + + # Handle pinning changes + if "pinned" in updates: + if updates["pinned"]: + await r.sadd("bulletin:pinned", post_id) + await r.zadd("bulletin:pinned_order", {post_id: post_dict.get("pin_order", 0)}) + else: + await r.srem("bulletin:pinned", post_id) + await r.zrem("bulletin:pinned_order", post_id) + + # Save updated post + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + + # Update Supabase + try: + from supabase import create_client + + supabase_url = os.getenv("SUPABASE_URL") + supabase_key = os.getenv("SUPABASE_SERVICE_KEY") + if supabase_url and supabase_key: + client = create_client(supabase_url, supabase_key) + client.table("bulletin_posts").upsert(post_dict).execute() + except Exception as e: + logger.error(f"Supabase bulletin post update failed: {e}") + + return Post(**post_dict) + + @staticmethod + async def delete_post(post_id: str) -> bool: + """Delete a post (soft delete - move to archive).""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return False + + post_dict = json.loads(data) + post_dict["status"] = "archived" + post_dict["archived_at"] = datetime.utcnow().isoformat() + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + await r.srem("bulletin:status:published", post_id) + await r.srem("bulletin:status:draft", post_id) + await r.srem("bulletin:status:review", post_id) + await r.sadd("bulletin:status:archived", post_id) + await r.srem("bulletin:pinned", post_id) + + return True + + @staticmethod + async def list_posts( + category: str | None = None, + status: str | None = None, + target_audience: str | None = None, + author_id: str | None = None, + pinned_only: bool = False, + search_query: str | None = None, + tags: list[str] | None = None, + limit: int = 50, + offset: int = 0, + sort_by: str = "created_at", + sort_order: str = "desc", + ) -> dict[str, Any]: + """List posts with filtering.""" + r = await BulletinBoardManager._get_redis() + + # Start with all posts or filtered set + if pinned_only: + post_ids = await r.smembers("bulletin:pinned") + elif category: + post_ids = await r.smembers(f"bulletin:category:{category}") + elif status: + post_ids = await r.smembers(f"bulletin:status:{status}") + elif author_id: + post_ids = await r.smembers(f"bulletin:author:{author_id}") + elif target_audience: + post_ids = await r.smembers(f"bulletin:audience:{target_audience}") + elif search_query: + # Search by words + words = re.findall(r"\w+", search_query.lower()) + if words: + sets = [f"bulletin:search:{w}" for w in words if len(w) > 2] + if sets: + post_ids = await r.sinter(sets) + else: + post_ids = set() + else: + post_ids = set() + else: + all_posts = await r.hgetall("rmi:bulletin_posts") + post_ids = set(all_posts.keys()) + + # Apply additional filters + if tags: + tagged_posts = set() + all_posts = await r.hgetall("rmi:bulletin_posts") + for pid, data in all_posts.items(): + post_dict = json.loads(data) + if any(tag in post_dict.get("tags", []) for tag in tags): + tagged_posts.add(pid) + post_ids = post_ids.intersection(tagged_posts) + + # Fetch and sort posts + posts = [] + all_posts = await r.hgetall("rmi:bulletin_posts") + for pid in post_ids: + data = all_posts.get(pid) + if data: + post_dict = json.loads(data) + posts.append(post_dict) + + # Sort + reverse = sort_order == "desc" + posts.sort(key=lambda x: x.get(sort_by, ""), reverse=reverse) + + # Pinned posts first if not pinned_only + if not pinned_only: + pinned_ids = await r.smembers("bulletin:pinned") + posts.sort( + key=lambda x: (x["post_id"] not in pinned_ids, x.get(sort_by, "")), + reverse=not reverse, + ) + + total = len(posts) + posts = posts[offset : offset + limit] + + return { + "posts": posts, + "total": total, + "limit": limit, + "offset": offset, + } + + @staticmethod + async def publish_scheduled() -> list[str]: + """Publish posts that are scheduled for now.""" + r = await BulletinBoardManager._get_redis() + now = int(time.time()) + + # Get scheduled posts that are due + due = await r.zrangebyscore("bulletin:scheduled", 0, now) + published = [] + + for post_id in due: + data = await r.hget("rmi:bulletin_posts", post_id) + if data: + post_dict = json.loads(data) + post_dict["status"] = "published" + post_dict["published_at"] = datetime.utcnow().isoformat() + post_dict["scheduled_at"] = None + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + await r.srem("bulletin:status:scheduled", post_id) + await r.sadd("bulletin:status:published", post_id) + await r.zrem("bulletin:scheduled", post_id) + + published.append(post_id) + + return published + + @staticmethod + async def archive_expired() -> list[str]: + """Archive posts that have expired.""" + r = await BulletinBoardManager._get_redis() + now = datetime.utcnow().isoformat() + + all_posts = await r.hgetall("rmi:bulletin_posts") + archived = [] + + for post_id, data in all_posts.items(): + post_dict = json.loads(data) + if post_dict.get("status") == "published" and post_dict.get("expires_at") and post_dict["expires_at"] < now: + post_dict["status"] = "archived" + post_dict["archived_at"] = now + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + await r.srem("bulletin:status:published", post_id) + await r.sadd("bulletin:status:archived", post_id) + await r.srem("bulletin:pinned", post_id) + + archived.append(post_id) + + return archived + + @staticmethod + async def add_comment( + post_id: str, + author_id: str, + author_name: str, + author_email: str, + content: str, + parent_id: str | None = None, + ) -> Comment | None: + """Add a comment to a post.""" + r = await BulletinBoardManager._get_redis() + + # Check if post exists and allows comments + post_data = await r.hget("rmi:bulletin_posts", post_id) + if not post_data: + return None + + post_dict = json.loads(post_data) + if not post_dict.get("allow_comments", False): + return None + + comment_id = f"comment_{int(time.time())}_{os.urandom(4).hex()}" + now = datetime.utcnow().isoformat() + + comment = Comment( + comment_id=comment_id, + post_id=post_id, + author_id=author_id, + author_name=author_name, + author_email=author_email, + content=ContentSanitizer.sanitize(content)[:2000], + created_at=now, + updated_at=now, + parent_id=parent_id, + ) + + await r.hset("rmi:bulletin_comments", comment_id, json.dumps(comment.to_dict())) + await r.sadd(f"bulletin:post_comments:{post_id}", comment_id) + + # Update post comment count + post_dict["comments_count"] = post_dict.get("comments_count", 0) + 1 + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + + return comment + + @staticmethod + async def get_comments(post_id: str, limit: int = 100) -> list[dict]: + """Get comments for a post.""" + r = await BulletinBoardManager._get_redis() + comment_ids = await r.smembers(f"bulletin:post_comments:{post_id}") + + comments = [] + for cid in comment_ids: + data = await r.hget("rmi:bulletin_comments", cid) + if data: + comments.append(json.loads(data)) + + comments.sort(key=lambda x: x.get("created_at", ""), reverse=True) + return comments[:limit] + + @staticmethod + async def get_stats() -> dict[str, Any]: + """Get bulletin board statistics.""" + r = await BulletinBoardManager._get_redis() + + stats = { + "total_posts": await r.hlen("rmi:bulletin_posts") or 0, + "published": await r.scard("bulletin:status:published") or 0, + "drafts": await r.scard("bulletin:status:draft") or 0, + "review": await r.scard("bulletin:status:review") or 0, + "archived": await r.scard("bulletin:status:archived") or 0, + "scheduled": await r.scard("bulletin:status:scheduled") or 0, + "pinned": await r.scard("bulletin:pinned") or 0, + "total_comments": await r.hlen("rmi:bulletin_comments") or 0, + "categories": {}, + } + + # Count by category + for cat in PostCategory: + count = await r.scard(f"bulletin:category:{cat.value}") or 0 + stats["categories"][cat.value] = count + + return stats + + @staticmethod + async def track_engagement(post_id: str, action: str) -> bool: + """Track engagement (view, click, etc.).""" + r = await BulletinBoardManager._get_redis() + data = await r.hget("rmi:bulletin_posts", post_id) + if not data: + return False + + post_dict = json.loads(data) + + if action == "view": + post_dict["view_count"] = post_dict.get("view_count", 0) + 1 + elif action == "click": + post_dict["click_count"] = post_dict.get("click_count", 0) + 1 + + # Calculate engagement score + views = post_dict.get("view_count", 0) + clicks = post_dict.get("click_count", 0) + post_dict["engagement_score"] = round((clicks / max(views, 1)) * 100, 2) + + await r.hset("rmi:bulletin_posts", post_id, json.dumps(post_dict)) + return True + + +# ═══════════════════════════════════════════════════════════ +# BADGES & X402 BOT PAYMENTS +# ═══════════════════════════════════════════════════════════ + +BADGES = { + "first_post": { + "name": "First Words", + "icon": "💬", + "desc": "Made your first post", + "tier": "bronze", + }, + "10_posts": {"name": "Chatterbox", "icon": "📢", "desc": "10 posts", "tier": "bronze"}, + "50_posts": {"name": "Board Regular", "icon": "🎙️", "desc": "50 posts", "tier": "silver"}, + "100_posts": { + "name": "Terminally Online", + "icon": "🖥️", + "desc": "100 posts - touch grass", + "tier": "gold", + }, + "10_upvotes": { + "name": "Approved", + "icon": "👍", + "desc": "10 upvotes on a post", + "tier": "bronze", + }, + "50_upvotes": {"name": "Crowd Favorite", "icon": "⭐", "desc": "50 upvotes", "tier": "silver"}, + "100_upvotes": {"name": "Legendary", "icon": "👑", "desc": "100 upvotes", "tier": "gold"}, + "rug_reporter": { + "name": "Rug Detective", + "icon": "🔍", + "desc": "Reported 3 verified rugs", + "tier": "silver", + }, + "scam_buster": { + "name": "Scam Buster", + "icon": "🛡️", + "desc": "10 scam alerts verified", + "tier": "gold", + }, + "honeypot_hunter": { + "name": "Honeypot Hunter", + "icon": "🍯", + "desc": "Found 5 honeypots", + "tier": "silver", + }, + "whale_watcher": { + "name": "Whale Watcher", + "icon": "🐋", + "desc": "Tracked 10 whale moves", + "tier": "silver", + }, + "alpha_caller": { + "name": "Alpha Caller", + "icon": "📈", + "desc": "Called 5 pumps", + "tier": "gold", + }, + "degen": { + "name": "Certified Degen", + "icon": "🎰", + "desc": "Posted in every category", + "tier": "gold", + }, + "rug_survivor": { + "name": "Rug Survivor", + "icon": "💀", + "desc": "Posted about getting rugged", + "tier": "bronze", + }, + "diamond_hands": { + "name": "Diamond Hands", + "icon": "💎", + "desc": "Held through -90%", + "tier": "diamond", + }, + "based": { + "name": "Based", + "icon": "🧠", + "desc": "Called a 10x before it happened", + "tier": "diamond", + }, + "bot_verified": { + "name": "Verified Bot", + "icon": "🤖", + "desc": "Registered x402 bot", + "tier": "silver", + }, + "bot_pro": {"name": "Bot Pro", "icon": "⚡", "desc": "100+ API calls", "tier": "gold"}, +} +BADGE_TIERS = {"bronze": "#CD7F32", "silver": "#C0C0C0", "gold": "#FFD700", "diamond": "#B9F2FF"} + + +async def get_user_badges(user_id: str) -> list: + try: + r = await BulletinBoardManager._get_redis() + ids = await r.smembers(f"bb:badges:{user_id}") + await r.close() + return [{"id": bid, **BADGES[bid]} for bid in ids if bid in BADGES] + except Exception: + return [] + + +async def award_badge(user_id: str, badge_id: str) -> bool: + if badge_id not in BADGES: + return False + try: + r = await BulletinBoardManager._get_redis() + ok = await r.sadd(f"bb:badges:{user_id}", badge_id) + await r.close() + return ok > 0 + except Exception: + return False + + +async def get_user_reputation(user_id: str) -> dict: + try: + r = await BulletinBoardManager._get_redis() + p = r.pipeline() + p.get(f"bb:rep:{user_id}") + p.scard(f"bb:badges:{user_id}") + p.get(f"bb:posts:{user_id}") + rep, bc, pc = await p.execute() + await r.close() + return {"reputation": int(rep or 0), "badge_count": bc or 0, "post_count": int(pc or 0)} + except Exception: + return {"reputation": 0, "badge_count": 0, "post_count": 0} + + +X402_BB_POST_PRICE = "$1.00" + + +async def verify_x402_bot(tx_hash: str, bot_addr: str) -> bool: + try: + r = await BulletinBoardManager._get_redis() + if await r.get(f"bb:x402:tx:{tx_hash}"): + await r.close() + return False + await r.setex(f"bb:x402:tx:{tx_hash}", 86400, bot_addr) + await r.setex(f"bb:x402:bot:{bot_addr}", 2592000, "1") # 30 day auth + await r.close() + return True + except Exception: + return False diff --git a/app/domains/databus/provider_implementations.py b/app/domains/databus/provider_implementations.py index 48f5a12..88e3a91 100644 --- a/app/domains/databus/provider_implementations.py +++ b/app/domains/databus/provider_implementations.py @@ -183,7 +183,10 @@ async def _passthrough_news(**kwargs) -> dict | None: async def _passthrough_alerts(**kwargs) -> dict | None: """Alerts from our real alert pipeline.""" try: - from app.alert_pipeline import get_active_alert_count, get_recent_alerts + from app.domains.intelligence.alert_pipeline import ( + get_active_alert_count, + get_recent_alerts, + ) count = await get_active_alert_count() recent = await get_recent_alerts(limit=kwargs.get("limit", 20)) diff --git a/app/domains/databus/providers/__init__.py b/app/domains/databus/providers/__init__.py index 3525891..e7d644e 100644 --- a/app/domains/databus/providers/__init__.py +++ b/app/domains/databus/providers/__init__.py @@ -362,7 +362,10 @@ async def _passthrough_news(**kwargs) -> dict | None: async def _passthrough_alerts(**kwargs) -> dict | None: """Alerts from our real alert pipeline.""" try: - from app.alert_pipeline import get_active_alert_count, get_recent_alerts + from app.domains.intelligence.alert_pipeline import ( + get_active_alert_count, + get_recent_alerts, + ) count = await get_active_alert_count() recent = await get_recent_alerts(limit=kwargs.get("limit", 20)) diff --git a/app/domains/databus/webhooks.py b/app/domains/databus/webhooks.py index 0574260..35d990c 100644 --- a/app/domains/databus/webhooks.py +++ b/app/domains/databus/webhooks.py @@ -152,7 +152,7 @@ async def handle_webhook(service: str, payload: dict, headers: dict, raw_body: b risk = _assess_webhook_risk(service, event_type, payload) if risk["level"] in ("HIGH", "CRITICAL"): try: - from app.alert_pipeline import push_alert + from app.domains.intelligence.alert_pipeline import push_alert await push_alert( title=f"[{service.upper()}] {risk['title']}", diff --git a/app/domains/intelligence/__init__.py b/app/domains/intelligence/__init__.py new file mode 100644 index 0000000..6123681 --- /dev/null +++ b/app/domains/intelligence/__init__.py @@ -0,0 +1,25 @@ +"""Intelligence domain - public API. + +Phase 4 domain consolidation: intelligence-related modules -> app.domains.intelligence. +""" +from __future__ import annotations + +from app.domains.intelligence.alert_pipeline import ( + get_active_alert_count, + get_recent_alerts, + push_alert, + run_alert_scan, + scan_known_scams, + scan_solana_new_pairs, +) +from app.domains.intelligence.intel_pipeline import run_intelligence_cycle + +__all__ = [ + "get_active_alert_count", + "get_recent_alerts", + "push_alert", + "run_alert_scan", + "run_intelligence_cycle", + "scan_known_scams", + "scan_solana_new_pairs", +] diff --git a/app/domains/intelligence/alert_pipeline.py b/app/domains/intelligence/alert_pipeline.py new file mode 100644 index 0000000..f93c7f9 --- /dev/null +++ b/app/domains/intelligence/alert_pipeline.py @@ -0,0 +1,370 @@ +""" +RMI Alert Pipeline - Real-time threat intelligence from scanners. +================================================================= +Feeds the Live Intel panel, WebSocket streams, and alert endpoints. + +Data sources: + - SENTINEL scanner (risk scores, scam detection) + - GoPlus security API (honeypot, tax, proxy checks) + - DexScreener new pairs (fresh launches to scan) + - Our own RAG scam patterns + - Wallet label cross-references + +Alert flow: + Scanner → alert_pipeline.push_alert() → Redis sorted set + pub/sub + Homepage reads from /api/v1/alerts/recent + Sidebar reads from /api/v1/alerts/count + WebSocket streams to connected clients +""" + +import json +import logging +import os +import time +from datetime import UTC, datetime + +logger = logging.getLogger("alert_pipeline") + +REDIS_HOST = os.getenv("REDIS_HOST", "rmi-redis") +REDIS_PORT = int(os.getenv("REDIS_PORT", "6379")) +REDIS_PW = os.getenv("REDIS_PASSWORD", "") + +ALERT_KEY = "rmi:alerts:recent" +ALERT_COUNT_KEY = "rmi:alerts:count:active" +ALERT_MAX = 500 # Keep last 500 alerts + + +async def _get_redis(): + """Get async Redis connection.""" + import redis.asyncio as aioredis + + return aioredis.Redis( + host=REDIS_HOST, + port=REDIS_PORT, + password=REDIS_PW or None, + decode_responses=True, + ) + + +async def get_active_alert_count() -> int: + """Get count of active (unacknowledged) alerts.""" + try: + r = await _get_redis() + count = await r.zcard(ALERT_KEY) + await r.close() + return count + except Exception as e: + logger.debug(f"Alert count error: {e}") + return 0 + + +async def push_alert( + alert_type: str, + title: str, + description: str = "", + severity: str = "high", + chain: str = "unknown", + token: str = "", + token_symbol: str = "", + wallet: str = "", + risk_score: int = 0, + metadata: dict | None = None, +) -> str: + """ + Push a new alert to the pipeline. + Returns alert_id. + """ + alert_id = f"alt_{int(time.time())}_{os.urandom(4).hex()}" + + alert = { + "id": alert_id, + "type": alert_type, + "title": title, + "description": description, + "severity": severity, + "chain": chain, + "token": token, + "token_symbol": token_symbol, + "wallet": wallet, + "risk_score": risk_score, + "acknowledged": False, + "created_at": datetime.now(UTC).isoformat(), + **(metadata or {}), + } + + try: + r = await _get_redis() + score = time.time() + await r.zadd(ALERT_KEY, {json.dumps(alert): score}) + # Trim old alerts + await r.zremrangebyrank(ALERT_KEY, 0, -(ALERT_MAX + 1)) + # Pub/sub for WebSocket streaming + await r.publish("rmi:ws:alerts", json.dumps(alert)) + await r.close() + logger.info(f"Alert pushed: {alert_type} | {title[:60]}") + except Exception as e: + logger.error(f"Failed to push alert: {e}") + + return alert_id + + +async def get_recent_alerts(limit: int = 20, severity: str = "") -> list[dict]: + """Get recent alerts with optional severity filter. Normalizes old/new formats.""" + try: + r = await _get_redis() + raw = await r.zrevrange(ALERT_KEY, 0, limit * 2 - 1) + await r.close() + + alerts = [] + for entry in raw: + try: + alert = json.loads(entry) + + # Normalize old alert format to new + if "title" not in alert: + alert["title"] = alert.get("message", alert.get("event", "Unknown alert")) + if "description" not in alert: + desc_parts = [] + if alert.get("symbol"): + desc_parts.append(f"Token: {alert['symbol']}") + if alert.get("risk_score"): + desc_parts.append(f"Risk: {alert['risk_score']}/100") + flags = alert.get("risk_flags", []) + if flags: + desc_parts.append("; ".join(str(f)[:80] for f in flags[:2])) + alert["description"] = " | ".join(desc_parts) + if "severity" not in alert: + score = alert.get("risk_score", 50) + alert["severity"] = "critical" if score >= 85 else "high" if score >= 65 else "medium" + if "chain" not in alert: + alert["chain"] = alert.get("chain", "unknown") + + if severity and alert.get("severity") != severity: + continue + alerts.append(alert) + if len(alerts) >= limit: + break + except json.JSONDecodeError: + pass + + return alerts + except Exception as e: + logger.error(f"get_recent_alerts error: {e}") + return [] + + +# ── Alert generators - called by cron jobs or on-demand ────────── + + +async def scan_solana_new_pairs(limit: int = 5) -> int: + """ + Scan latest Solana pairs from DexScreener for scam patterns. + Pushes alerts for high-risk tokens. + Returns number of alerts generated. + """ + import httpx + + pushed = 0 + + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.get("https://api.dexscreener.com/latest/dex/search", params={"q": "SOL USDC"}) + if resp.status_code != 200: + return 0 + + data = resp.json() + pairs = data.get("pairs", [])[:limit] + + for pair in pairs: + token_addr = pair.get("baseToken", {}).get("address", "") + token_name = pair.get("baseToken", {}).get("name", "Unknown") + token_symbol = pair.get("baseToken", {}).get("symbol", "???") + liquidity = float(pair.get("liquidity", {}).get("usd", 0) or 0) + volume = float(pair.get("volume", {}).get("h24", 0) or 0) + price_change = float(pair.get("priceChange", {}).get("h24", 0) or 0) + created_at = pair.get("pairCreatedAt", 0) + age_hours = (time.time() - created_at / 1000) / 3600 if created_at else 999 + + # Risk heuristics + risk_flags = [] + + if liquidity < 1000: + risk_flags.append("low_liquidity") + if volume == 0: + risk_flags.append("no_volume") + if price_change < -80: + risk_flags.append(f"crashed_{abs(price_change):.0f}%") + if age_hours < 1 and liquidity < 5000: + risk_flags.append("fresh_launch_low_liq") + if price_change > 500: + risk_flags.append(f"pumped_{price_change:.0f}%") + + if risk_flags: + await push_alert( + alert_type="new_pair_risk", + title=f"{token_symbol}: {' | '.join(risk_flags[:2])}", + description=f"New pair {token_name} ({token_symbol}) on Solana. " + f"Liquidity: ${liquidity:,.0f}, Age: {age_hours:.1f}h, " + f"24h change: {price_change:+.1f}%", + severity="critical" if len(risk_flags) >= 3 else "high", + chain="solana", + token=token_addr, + token_symbol=token_symbol, + risk_score=min(90, len(risk_flags) * 25), + metadata={ + "risk_flags": risk_flags, + "liquidity_usd": liquidity, + "age_hours": age_hours, + }, + ) + pushed += 1 + + return pushed + except Exception as e: + logger.warning(f"Solana scan error: {e}") + return pushed + + +async def scan_known_scams(limit: int = 3) -> int: + """ + Check our RAG known_scams collection for recently added entries. + Pushes alerts for new scam patterns detected. + """ + pushed = 0 + try: + r = await _get_redis() + # Check for recent scam pattern additions + scam_ids = await r.smembers("rag:idx:known_scams") + + recent_count = 0 + for sid in list(scam_ids)[:20]: + doc = await r.get(f"rag:known_scams:{sid}") + if doc: + try: + data = json.loads(doc) + added = data.get("metadata", {}).get("added_at", "") + if added: + age_h = (time.time() - datetime.fromisoformat(added.replace("Z", "+00:00")).timestamp()) / 3600 + if age_h < 24: + recent_count += 1 + except Exception: + pass + + await r.close() + + if recent_count > 0: + await push_alert( + alert_type="scam_pattern_update", + title=f"{recent_count} new scam patterns detected in last 24h", + description="New rug pull, honeypot, or phishing patterns added to the knowledge base.", + severity="high", + chain="multi", + risk_score=85, + ) + pushed += 1 + + return pushed + except Exception as e: + logger.warning(f"Known scams scan error: {e}") + return pushed + + +async def run_alert_scan() -> dict[str, int]: + """ + Run a full alert scan across all sources. + Called by cron job every 15 minutes. + """ + results = {} + + # Scan Solana new pairs + results["solana_pairs"] = await scan_solana_new_pairs(limit=8) + + # Scan known scams + results["known_scams"] = await scan_known_scams() + + total = sum(results.values()) + logger.info(f"Alert scan complete: {total} new alerts ({results})") + return results + + +# ── Seed some initial alerts if Redis is empty ──────────────────── + + +async def seed_initial_alerts(): + """Seed baseline alerts so the system isn't empty on first run.""" + r = await _get_redis() + existing = await r.zcard(ALERT_KEY) + await r.close() + + if existing > 0: + return # Already has alerts + + seeds = [ + ( + "honeypot", + "Honeypot detected on Base: 0xdead...", + "Token has sell restrictions and blacklist. Buyers cannot exit.", + "critical", + "base", + ), + ( + "whale", + "Whale moved 5M USDC to Binance", + "Wallet 0xABCD... transferred $5M USDC to Binance hot wallet. Possible sell pressure.", + "high", + "ethereum", + ), + ( + "rugpull", + "Liquidity removed from $SCAM token", + "100% of liquidity pool withdrawn by deployer. Token is now worthless.", + "critical", + "solana", + ), + ( + "bundler", + "Sniper bundle detected: $NEWLAUNCH", + "Coordinated wallet cluster bought 60% of supply in first block.", + "high", + "solana", + ), + ( + "contract", + "Unverified proxy contract found", + "Token uses upgradeable proxy with unverified implementation. Owner can change logic.", + "high", + "base", + ), + ( + "concentration", + "90% supply held by 3 wallets on $DANGER", + "Extreme holder concentration. Classic rug pull setup.", + "critical", + "ethereum", + ), + ( + "wash_trade", + "Wash trading detected on $FAKEVOL", + "95% of volume is self-trading between linked wallets.", + "high", + "bsc", + ), + ( + "phishing", + "Fake airdrop targeting $BONK holders", + "Phishing site detected posing as official BONK airdrop. Users losing funds.", + "critical", + "solana", + ), + ] + + for alert_type, title, desc, severity, chain in seeds: + await push_alert( + alert_type=alert_type, + title=title, + description=desc, + severity=severity, + chain=chain, + ) + + logger.info(f"Seeded {len(seeds)} initial alerts") diff --git a/app/domains/intelligence/intel_pipeline.py b/app/domains/intelligence/intel_pipeline.py new file mode 100644 index 0000000..ff2c30f --- /dev/null +++ b/app/domains/intelligence/intel_pipeline.py @@ -0,0 +1,298 @@ +""" +RMI Intelligence Pipeline - HF + Supabase + RAG +================================================ +Unified intelligence service: scam classification (HF models), +wallet labeling via RAG pattern matching, Supabase hybrid storage. +""" + +import hashlib +import logging +import os +from datetime import UTC, datetime + +import httpx +from dotenv import load_dotenv + +load_dotenv("/app/.env", override=True) + +logger = logging.getLogger(__name__) + +HF_TOKEN = os.getenv("HF_TOKEN", "") +HF_API = "https://api-inference.huggingface.co/models" + + +def _get_url(): + return os.getenv("SUPABASE_URL", "") + + +def _get_key(): + return os.getenv("SUPABASE_SERVICE_KEY", "") + + +def _get_headers(): + key = _get_key() + return { + "apikey": key, + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + + +# ── HF Models (Paywalled - using local fallback) ────── +# HF Inference API now requires PRO subscription ($9/mo). +# Using local pattern matching + RAG memory instead. +# Enable HF by setting HUGGINGFACE_TOKEN to a valid PRO key. + +SCAM_LABELS = ["scam", "rugpull", "honeypot", "legitimate", "phishing", "ponzi"] + +SCAM_KEYWORDS = { + "scam": ["scam", "fraud", "stole", "stolen", "exit scam", "fake"], + "rugpull": ["rug", "rugpull", "pulled liquidity", "drained", "removed liquidity", "lp removed"], + "honeypot": [ + "honeypot", + "cannot sell", + "can't sell", + "unable to sell", + "transfer disabled", + "sell tax 100", + ], + "phishing": [ + "phishing", + "airdrop scam", + "claim reward", + "verify wallet", + "seed phrase", + "private key", + ], + "ponzi": [ + "ponzi", + "mlm", + "multi level", + "referral rewards", + "guaranteed returns", + "double your", + ], + "insider": ["insider", "team wallet", "dev wallet", "pre-sale", "unlocked tokens", "vesting"], +} + + +async def classify_scam_risk(text: str) -> dict: + """Classify scam risk using local pattern matching (HF paywalled).""" + text_lower = text.lower() + scores = {} + + for label, keywords in SCAM_KEYWORDS.items(): + score = sum(1 for kw in keywords if kw in text_lower) + if score > 0: + scores[label] = min(score / len(keywords), 1.0) + + try: + from app.rag_service import search_documents + + rag_results = await search_documents("known_scams", text, limit=3) + for r in rag_results: + content = r.get("content", "").lower() + for label, keywords in SCAM_KEYWORDS.items(): + if any(kw in content for kw in keywords): + scores[label] = scores.get(label, 0) + 0.1 + except Exception: + pass + + if not scores: + return { + "risk": "low", + "labels": {}, + "confidence": 0, + "is_scam": False, + "risk_level": "low", + "model": "local_pattern_match", + } + + top = max(scores, key=scores.get) + top_score = scores[top] + is_scam = top in ("scam", "rugpull", "honeypot", "phishing", "ponzi") + + return { + "model": "local_pattern_match", + "labels": {k: round(v, 3) for k, v in sorted(scores.items(), key=lambda x: x[1], reverse=True)}, + "top_label": top, + "confidence": round(top_score, 3), + "is_scam": is_scam, + "risk_level": "high" if (is_scam and top_score > 0.5) else "medium" if is_scam else "low", + } + + +async def generate_embedding(text: str) -> list[float] | None: + """Generate embedding vector for RAG storage using HF free tier.""" + if not HF_TOKEN: + return None + + async with httpx.AsyncClient(timeout=60) as client: + r = await client.post( + f"{HF_API}/{EMBEDDING_MODEL}", # noqa: F821 -- pre-existing bug, see fix(f821) tracking issue + headers={"Authorization": f"Bearer {HF_TOKEN}"}, + json={"inputs": text[:1024]}, + ) + if r.status_code == 503: + logger.warning("HF embedding cold start, model loading") + return None + if r.status_code == 200: + result = r.json() + # Handle different response formats + if isinstance(result, list) and len(result) > 0: + return result[0] if isinstance(result[0], list) else result + return result + logger.warning(f"HF embedding failed: {r.status_code}") + return None + + +# ── Wallet Labeling via Pattern Memory ────────────────── + +WALLET_PATTERNS = { + "sybil_farmer": [ + "funded from exchange within seconds of 100+ other wallets", + "identical funding amounts across multiple wallets", + "no organic activity, only test transactions", + "funded by known Sybil distributor", + ], + "wash_trader": [ + "circular transfers between related wallets", + "buys and sells same token within minutes", + "volume spikes without holder count changes", + "coordinated buy/sell patterns", + ], + "sandwich_bot": [ + "high frequency trading with frontrun pattern", + "buys before large buys, sells immediately after", + "MEV extraction patterns", + "flashbots bundle usage", + ], + "liquidity_remover": [ + "large LP removal shortly after token launch", + "multiple LP positions removed simultaneously", + "liquidity drained to fresh wallet", + "LP removal preceded by marketing push", + ], + "honeypot_deployer": [ + "deploys tokens that can't be sold", + "reuses contract code across multiple tokens", + "disables transfers after liquidity added", + "ownership not renounced, hidden mint functions", + ], + "phishing_operator": [ + "sends tokens to many addresses with scam links", + "impersonates legitimate token contracts", + "uses airdrop as phishing vector", + "connects to known phishing domains", + ], + "mixer_user": [ + "funds pass through Tornado Cash or similar", + "receives from mixer, sends to clean wallet", + "layered mixing through multiple hops", + "funds originate from high-risk sources", + ], + "insider_trader": [ + "buys tokens before public announcements", + "linked to team wallets or deployers", + "sells immediately after hype peak", + "coordinated timing with other insiders", + ], +} + + +async def label_wallet(wallet_data: dict) -> dict: + """Label a wallet based on behavioral patterns and RAG memory.""" + labels = [] + confidence_scores = {} + + # Check behavioral patterns + behavior = wallet_data.get("behavior_summary", "") + wallet_data.get("transactions", []) + + for label, patterns in WALLET_PATTERNS.items(): + score = 0 + for pattern in patterns: + if pattern.lower() in behavior.lower(): + score += 1 + if score > 0: + confidence = min(score / len(patterns), 1.0) + confidence_scores[label] = round(confidence, 2) + if confidence > 0.3: + labels.append({"label": label, "confidence": round(confidence, 2)}) + + # Query RAG for similar wallet patterns + try: + from app.rag_service import search_documents + + rag_results = await search_documents( + "wallet_profiles", wallet_data.get("address", "") + " " + behavior, limit=5 + ) + if rag_results: + for r in rag_results: + content = r.get("content", "") + for label, patterns in WALLET_PATTERNS.items(): + if any(p.lower() in content.lower() for p in patterns): + if label not in confidence_scores: + confidence_scores[label] = 0 + confidence_scores[label] = min(confidence_scores[label] + 0.15, 1.0) + except Exception: + pass + + labels = [ + {"label": k, "confidence": v} + for k, v in sorted(confidence_scores.items(), key=lambda x: x[1], reverse=True) + if v > 0.2 + ] + + return { + "wallet_address": wallet_data.get("address", "unknown"), + "labels": labels[:5], + "primary_label": labels[0]["label"] if labels else "unknown", + "risk_score": max([line["confidence"] for line in labels]) * 100 if labels else 0, + "analyzed_at": datetime.now(UTC).isoformat(), + } + + +# ── Supabase Hybrid Storage ──────────────────────────── + + +async def sync_to_supabase(collection: str, document: dict) -> dict: + """Sync RAG document to Supabase for persistent hybrid storage.""" + if not _get_url() or not _get_key(): + return {"status": "skipped", "reason": "No Supabase config"} + + doc_id = hashlib.sha256(f"{collection}:{document.get('content', '')[:100]}".encode()).hexdigest()[:16] + + payload = { + "document_id": doc_id, + "collection": collection, + "content": document.get("content", "")[:5000], + "metadata": document.get("metadata", {}), + "synced_at": datetime.now(UTC).isoformat(), + } + + headers = _get_headers() + headers["Prefer"] = "resolution=merge-duplicates" + + async with httpx.AsyncClient(timeout=15) as client: + r = await client.post(f"{_get_url()}/rest/v1/rag_documents", json=payload, headers=headers) + if r.status_code in (200, 201, 409): + return {"status": "synced", "doc_id": doc_id, "supabase_status": r.status_code} + return {"status": "failed", "error": r.text[:200]} + + +# ── Batch Processing ─────────────────────────────────── + + +async def run_intelligence_cycle() -> dict: + """Run a full intelligence cycle: classify, label, embed, sync.""" + results = { + "cycle": datetime.now(UTC).isoformat(), + "scam_checks": 0, + "wallet_labels": 0, + "embeddings": 0, + "supabase_syncs": 0, + "errors": [], + } + + return results diff --git a/app/domains/intelligence/meme_intelligence.py b/app/domains/intelligence/meme_intelligence.py new file mode 100644 index 0000000..7201df4 --- /dev/null +++ b/app/domains/intelligence/meme_intelligence.py @@ -0,0 +1,423 @@ +""" +Meme Intelligence Platform - Meme Coin Tracking, Smart Money in Memes, +Big Wins/Losses, KOL Scorecards, Social Monitoring. + +Integrations: +- DexScreener: meme token launches, trending +- LunarCrush: social sentiment, social dominance +- X/Twitter: KOL posts, viral content +- Telegram: channel monitoring, group sentiment +- Arkham: whale tracking in memes +- Birdeye: Solana meme tokens +- Pump.fun: new meme launches +""" + +import logging +import os + +from dotenv import load_dotenv + +load_dotenv("/app/.env", override=True) +from datetime import UTC, datetime, timedelta # noqa: E402 + +logger = logging.getLogger(__name__) + +# ── Meme Intelligence Data Structures ───────────────────────── + +MEME_CHAINS = ["solana", "ethereum", "base", "bsc", "arbitrum"] + +MEME_CATEGORIES = { + "dog": ["dog", "doge", "shib", "akita", "kishu"], + "cat": ["cat", "pepe", "mog", "meow"], + "politi": ["trump", "biden", "maga", "politics"], + "celeb": ["celeb", "influencer", "famous"], + "ai": ["ai", "gpt", "neural", "chat"], + "gaming": ["game", "gaming", "nft", "metaverse"], + "other": [], +} + +# KOL Database Schema +KOL_DATABASE = { + # Example structure - populate from research + "twitter_handles": [], + "telegram_channels": [], + "wallet_addresses": [], + "track_record": {}, # past calls, win rate + "follower_counts": {}, + "engagement_rates": {}, +} + +# ── Meme Token Intelligence ──────────────────────────────────── + + +async def get_meme_trending(limit: int = 50) -> list[dict]: + """Get trending meme tokens across chains.""" + from app.unified_provider import get_unified_provider + + provider = get_unified_provider() + memes = [] + + for chain in MEME_CHAINS: + try: + # Get trending from DexScreener + trending = await provider.get_dexscreener_trending(chain) + + for token in trending[:20]: + # Classify as meme based on name/symbol + category = classify_meme_category( + token.get("baseToken", {}).get("symbol", ""), + token.get("baseToken", {}).get("name", ""), + ) + + if category != "other": + memes.append( + { + "address": token.get("baseToken", {}).get("address"), + "symbol": token.get("baseToken", {}).get("symbol"), + "name": token.get("baseToken", {}).get("name"), + "chain": chain, + "category": category, + "price_usd": token.get("priceUsd"), + "volume_24h": token.get("volume", {}).get("h24"), + "price_change_24h": token.get("priceChange", {}).get("h24"), + "liquidity_usd": token.get("liquidity", {}).get("usd"), + "fdv": token.get("fdv"), + "pair_age_hours": token.get("pairCreatedAt"), + "is_meme": True, + } + ) + except Exception as e: + logger.debug(f"Error getting meme trending for {chain}: {e}") + + # Sort by volume + memes.sort(key=lambda x: x.get("volume_24h", 0), reverse=True) + + return memes[:limit] + + +async def get_smart_money_in_memes(limit: int = 20) -> list[dict]: + """Track known smart money wallets trading memes.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Query for smart money activities in meme tokens + result = ( + supabase.table("smart_money_activities") + .select(""" + *, + wallets ( + wallet_address, + wallet_label, + wallet_category, + win_rate, + total_pnl_usd + ) + """) + .eq("is_meme", True) + .order("amount_usd", desc=True) + .limit(limit) + .execute() + ) + + return result.data or [] + + +async def get_meme_wins_losses(period: str = "24h") -> dict: + """Get biggest wins and losses in meme tokens.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Calculate time range + if period == "24h": + time_range = datetime.now(UTC) - timedelta(hours=24) + elif period == "7d": + time_range = datetime.now(UTC) - timedelta(days=7) + elif period == "30d": + time_range = datetime.now(UTC) - timedelta(days=30) + else: + time_range = datetime.now(UTC) - timedelta(hours=24) + + # Get biggest wins + wins = ( + supabase.table("whale_movements") + .select("*") + .gte("collected_at", time_range.isoformat()) + .eq("transaction_type", "sell") + .order("amount_usd", desc=True) + .limit(20) + .execute() + ) + + # Get biggest losses + losses = ( + supabase.table("whale_movements") + .select("*") + .gte("collected_at", time_range.isoformat()) + .eq("transaction_type", "buy") + .order("amount_usd", desc=True) + .limit(20) + .execute() + ) + + return { + "period": period, + "wins": wins.data or [], + "losses": losses.data or [], + } + + +def classify_meme_category(symbol: str, name: str) -> str: + """Classify meme token into category.""" + text = f"{symbol} {name}".lower() + + for category, keywords in MEME_CATEGORIES.items(): + if any(kw in text for kw in keywords): + return category + + return "other" + + +# ── KOL Intelligence ────────────────────────────────────────── + + +async def get_kol_scorecard(kol_handle: str) -> dict | None: + """Get KOL scorecard with track record.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Get KOL profile + result = supabase.table("kols").select("*").eq("twitter_handle", kol_handle).execute() + + if not result.data: + return None + + kol = result.data[0] + + # Get their past calls + calls = ( + supabase.table("kol_calls") + .select("*") + .eq("kol_id", kol["id"]) + .order("called_at", desc=True) + .limit(50) + .execute() + ) + + # Calculate stats + total_calls = len(calls.data) if calls.data else 0 + winning_calls = len([c for c in (calls.data or []) if c.get("pnl_pct", 0) > 0]) + win_rate = (winning_calls / total_calls * 100) if total_calls > 0 else 0 + + avg_pnl = sum([c.get("pnl_pct", 0) for c in (calls.data or [])]) / total_calls if total_calls > 0 else 0 + + return { + "kol": kol, + "stats": { + "total_calls": total_calls, + "winning_calls": winning_calls, + "win_rate": round(win_rate, 2), + "average_pnl": round(avg_pnl, 2), + "follower_count": kol.get("follower_count"), + "engagement_rate": kol.get("engagement_rate"), + }, + "recent_calls": calls.data[:10] if calls.data else [], + } + + +async def get_top_kols_by_category(category: str = "memes", limit: int = 20) -> list[dict]: + """Get top KOLs by category with scorecards.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + result = ( + supabase.table("kols") + .select("*") + .eq("primary_category", category) + .order("win_rate", desc=True) + .order("follower_count", desc=True) + .limit(limit) + .execute() + ) + + return result.data or [] + + +# ── Social Monitoring ───────────────────────────────────────── + + +async def monitor_social_sentiment(token_address: str) -> dict: + """Monitor social sentiment for a token.""" + # LunarCrush integration + try: + from app.lunarcrush_connector import get_lunarcrush_connector + + lc = get_lunarcrush_connector() + + sentiment = await lc.get_sentiment(token_address) + + return { + "token": token_address, + "social_volume": sentiment.get("social_volume"), + "social_dominance": sentiment.get("social_dominance"), + "sentiment_score": sentiment.get("sentiment_score"), + "mentions_24h": sentiment.get("mentions_24h"), + "mentions_change_24h": sentiment.get("mentions_change_24h"), + } + except Exception: + return {"error": "LunarCrush not available"} + + +async def get_viral_crypto_posts(hours: int = 24) -> list[dict]: + """Get viral crypto posts from X/Twitter.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + time_range = datetime.now(UTC) - timedelta(hours=hours) + + result = ( + supabase.table("viral_posts") + .select("*") + .gte("posted_at", time_range.isoformat()) + .order("engagement_score", desc=True) + .limit(50) + .execute() + ) + + return result.data or [] + + +# ── Hack/Drain Alerts ──────────────────────────────────────── + + +async def get_recent_hacks_drains(hours: int = 24) -> list[dict]: + """Get recent hacks and drains from monitoring.""" + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + time_range = datetime.now(UTC) - timedelta(hours=hours) + + result = ( + supabase.table("security_alerts") + .select("*") + .gte("detected_at", time_range.isoformat()) + .in_("alert_type", ["hack", "drain", "exploit", "rugpull"]) + .order("amount_usd", desc=True) + .limit(50) + .execute() + ) + + return result.data or [] + + +async def format_hack_alert_for_social(hack_data: dict) -> dict: + """Format hack alert for X/Telegram posting.""" + return { + "x_post": f"""🚨 HACK ALERT 🚨 + +Protocol: {hack_data.get("protocol_name", "Unknown")} +Amount: ${hack_data.get("amount_usd", 0):,.0f} +Chain: {hack_data.get("chain", "Unknown")} +Type: {hack_data.get("attack_type", "Unknown")} + +{hack_data.get("description", "")[:200]} + +#CryptoSecurity #DeFi #HackAlert""", + "telegram_post": f"""🚨 *HACK ALERT* 🚨 + +*Protocol:* {hack_data.get("protocol_name", "Unknown")} +*Amount:* ${hack_data.get("amount_usd", 0):,.0f} +*Chain:* {hack_data.get("chain", "Unknown")} +*Type:* {hack_data.get("attack_type", "Unknown")} + +{hack_data.get("description", "")[:500]} + +Stay safe out there! 🔒""", + "severity": hack_data.get("severity", "medium"), + "amount_usd": hack_data.get("amount_usd", 0), + } + + +# ── Wallet Screenshot Generation ────────────────────────────── + + +async def generate_wallet_screenshot(wallet_address: str, pnl_data: dict) -> str: + """Generate wallet PnL screenshot for sharing.""" + # This would use a graphics API (Alibaba, etc.) to generate images + # For now, return placeholder + + return { + "wallet": wallet_address, + "total_pnl": pnl_data.get("total_pnl_usd", 0), + "win_rate": pnl_data.get("win_rate", 0), + "top_wins": pnl_data.get("top_wins", []), + "top_losses": pnl_data.get("top_losses", []), + "image_url": f"/api/v1/images/wallet/{wallet_address}/pnl-summary", + "share_url": f"https://rugmunch.io/wallet/{wallet_address}", + } + + +# ── Content Generation ──────────────────────────────────────── + + +async def generate_marketing_content(content_type: str, data: dict) -> dict: + """Generate marketing content using AI.""" + # This would call Alibaba's AI API for content generation + + templates = { + "win_announcement": """ +🎉 BIG WIN ALERT! 🎉 + +Wallet: {wallet_address[:8]}...{wallet_address[-6:]} +Token: {token_symbol} +Profit: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) + +This whale called it early and rode it all the way up! 🐋 + +Track smart money: https://rugmunch.io/wallet/{wallet_address} + +#Crypto #MemeCoin #SmartMoney +""", + "loss_announcement": """ +💀 LOSS PORN 💀 + +Wallet: {wallet_address[:8]}...{wallet_address[-6:]} +Token: {token_symbol} +Loss: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) + +Oof. Another reminder to take profits! 📉 + +Learn from their mistakes: https://rugmunch.io/wallet/{wallet_address} + +#Crypto #Trading #LossPorn +""", + "kol_scorecard": """ +📊 KOL SCORECARD: @{kol_handle} + +Win Rate: {win_rate:.1f}% +Total Calls: {total_calls} +Avg PnL: {avg_pnl:.1f}% +Followers: {follower_count:,} + +Track their calls: https://rugmunch.io/kol/{kol_handle} + +#CryptoTwitter #KOL #Alpha +""", + } + + template = templates.get(content_type, "") + + # Format template with data + content = template.format(**data) if template else "" + + return { + "content_type": content_type, + "x_post": content[:280], + "telegram_post": content, + "generated_at": datetime.now(UTC).isoformat(), + } diff --git a/app/domains/intelligence/n8n_intelligence_workflows.py b/app/domains/intelligence/n8n_intelligence_workflows.py new file mode 100644 index 0000000..763ec33 --- /dev/null +++ b/app/domains/intelligence/n8n_intelligence_workflows.py @@ -0,0 +1,438 @@ +""" +n8n Workflow Automation - Market Intelligence & Premium Data Pulls. +Scheduled workflows for efficient data collection from multiple sources. +Runs every 30-60 minutes based on scan volume and data freshness needs. + +Integrations: +- CoinGecko: trending, global metrics, top gainers/losers +- DexScreener: new pairs, trending tokens, volume spikes +- Dune Analytics: custom queries for whale tracking, scam patterns +- Helius: token mints, large transfers, new token deployments +- Moralis: EVM whale movements, new contract deployments +- Arkham: entity labeling, exchange flows +- Nansen: smart money tracking (if available) +- Forta: real-time threat detection +""" + +import logging +from datetime import UTC, datetime + +logger = logging.getLogger(__name__) + +# ── n8n Workflow Definitions ───────────────────────────────── + +N8N_BASE_URL = "https://n8n.rugmunch.io" +N8N_WEBHOOK_URL = f"{N8N_BASE_URL}/webhook" + +# Market Intelligence Workflows +MARKET_INTEL_WORKFLOWS = { + "trending_tokens": { + "name": "Trending Tokens Collector", + "schedule": "*/30 * * * *", # Every 30 minutes + "sources": ["coingecko", "dexscreener", "geckoterminal"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/trending", + "description": "Collect trending tokens from multiple DEXs", + "output_table": "market_trending_tokens", + }, + "whale_movements": { + "name": "Whale Movement Tracker", + "schedule": "*/15 * * * *", # Every 15 minutes (high priority) + "sources": ["helius", "moralis", "arkham"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/whales", + "description": "Track large transfers across chains", + "output_table": "whale_movements", + }, + "new_token_deployments": { + "name": "New Token Deployments", + "schedule": "*/10 * * * *", # Every 10 minutes (fast detection) + "sources": ["helius", "moralis", "dexscreener"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/new-tokens", + "description": "Detect new token deployments in real-time", + "output_table": "new_token_deployments", + }, + "volume_spikes": { + "name": "Volume Spike Detector", + "schedule": "*/5 * * * *", # Every 5 minutes (critical) + "sources": ["dexscreener", "geckoterminal", "birdeye"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/volume-spikes", + "description": "Detect unusual volume increases", + "output_table": "volume_spikes", + }, + "global_metrics": { + "name": "Market Global Metrics", + "schedule": "0 * * * *", # Every hour + "sources": ["coingecko"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/global", + "description": "Global crypto market metrics", + "output_table": "market_global_metrics", + }, + "defi_protocols": { + "name": "DeFi Protocol Analytics", + "schedule": "0 */2 * * *", # Every 2 hours + "sources": ["defillama", "dune"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/defi", + "description": "DeFi TVL, volume, user metrics", + "output_table": "defi_protocol_metrics", + }, + "nft_market": { + "name": "NFT Market Intelligence", + "schedule": "0 */4 * * *", # Every 4 hours + "sources": ["alchemy", "opensea_api"], + "endpoint": f"{N8N_WEBHOOK_URL}/market/nft", + "description": "NFT floor prices, volume, trends", + "output_table": "nft_market_metrics", + }, +} + +# Premium Intelligence Workflows +PREMIUM_INTEL_WORKFLOWS = { + "smart_money_tracking": { + "name": "Smart Money Tracker", + "schedule": "*/30 * * * *", # Every 30 minutes + "sources": ["nansen", "arkham", "dune"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/smart-money", + "description": "Track known smart money wallets", + "output_table": "smart_money_activities", + "tier": "premium", + }, + "exchange_flows": { + "name": "Exchange Flow Analysis", + "schedule": "*/15 * * * *", # Every 15 minutes + "sources": ["arkham", "dune", "glassnode"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/exchange-flows", + "description": "Track exchange inflows/outflows", + "output_table": "exchange_flows", + "tier": "premium", + }, + "insider_trading": { + "name": "Insider Trading Detection", + "schedule": "*/20 * * * *", # Every 20 minutes + "sources": ["dune", "arkham", "helius"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/insider", + "description": "Detect potential insider trading patterns", + "output_table": "insider_trading_alerts", + "tier": "premium_plus", + }, + "launchpad_monitor": { + "name": "Launchpad Monitor", + "schedule": "*/10 * * * *", # Every 10 minutes + "sources": ["dexscreener", "pumpfun_api", "geckoterminal"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/launchpad", + "description": "Monitor new launches across platforms", + "output_table": "launchpad_monitoring", + "tier": "premium", + }, + "cluster_analysis": { + "name": "Cluster Pattern Analysis", + "schedule": "0 */2 * * *", # Every 2 hours + "sources": ["internal_clustering", "dune"], + "endpoint": f"{N8N_WEBHOOK_URL}/premium/clusters", + "description": "Deep cluster pattern analysis", + "output_table": "cluster_patterns", + "tier": "premium_plus", + }, +} + +# Security Intelligence Workflows +SECURITY_INTEL_WORKFLOWS = { + "scam_detection": { + "name": "Real-time Scam Detection", + "schedule": "*/5 * * * *", # Every 5 minutes (critical) + "sources": ["forta", "internal_ml", "community_reports"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/scams", + "description": "Detect new scam patterns", + "output_table": "scam_alerts", + "priority": "critical", + }, + "rugpull_detection": { + "name": "Rugpull Detection", + "schedule": "*/2 * * * *", # Every 2 minutes (ultra-critical) + "sources": ["internal_ml", "liquidity_monitor"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/rugpulls", + "description": "Detect rugpull patterns in real-time", + "output_table": "rugpull_alerts", + "priority": "critical", + }, + "contract_vulnerabilities": { + "name": "Contract Vulnerability Scanner", + "schedule": "0 * * * *", # Every hour + "sources": ["slither", "mythril", "internal_scanner"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/vulnerabilities", + "description": "Scan new contracts for vulnerabilities", + "output_table": "contract_vulnerabilities", + "priority": "high", + }, + "phishing_detection": { + "name": "Phishing Domain Detection", + "schedule": "0 */6 * * *", # Every 6 hours + "sources": ["guardian", "cryptoscamdb", "community"], + "endpoint": f"{N8N_WEBHOOK_URL}/security/phishing", + "description": "Detect new phishing domains", + "output_table": "phishing_domains", + "priority": "high", + }, +} + + +# ── n8n Workflow Execution ──────────────────────────────────── + + +async def trigger_n8n_workflow(workflow_name: str, data: dict | None = None) -> bool: + """Trigger an n8n workflow via webhook.""" + import httpx + + # Find workflow config + all_workflows = { + **MARKET_INTEL_WORKFLOWS, + **PREMIUM_INTEL_WORKFLOWS, + **SECURITY_INTEL_WORKFLOWS, + } + workflow = all_workflows.get(workflow_name) + + if not workflow: + logger.error(f"Workflow not found: {workflow_name}") + return False + + webhook_url = workflow["endpoint"] + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post( + webhook_url, + json={ + "workflow": workflow_name, + "triggered_at": datetime.now(UTC).isoformat(), + "data": data or {}, + }, + ) + + if response.status_code in (200, 201, 202): + logger.info(f"Triggered workflow: {workflow_name}") + return True + else: + logger.error(f"Workflow trigger failed: {workflow_name} - {response.status_code}") + return False + + except Exception as e: + logger.error(f"Workflow trigger error: {workflow_name} - {e}") + return False + + +async def execute_market_intelligence_pull(workflow_name: str): + """Execute a market intelligence data pull.""" + workflow = MARKET_INTEL_WORKFLOWS.get(workflow_name) + if not workflow: + return + + logger.info(f"Executing market intel pull: {workflow['name']}") + + # Collect data from sources + collected_data = { + "workflow": workflow_name, + "sources": workflow["sources"], + "collected_at": datetime.now(UTC).isoformat(), + "data": {}, + } + + # Source-specific collection logic + for source in workflow["sources"]: + try: + if source == "coingecko": + from app.coingecko_connector import get_coingecko_connector + + connector = get_coingecko_connector() + + if "trending" in workflow_name.lower(): + trending = await connector.get_trending() + collected_data["data"]["coingecko_trending"] = trending + elif "global" in workflow_name.lower(): + global_data = await connector.get_global_metrics() + collected_data["data"]["coingecko_global"] = global_data + + elif source == "dexscreener": + from app.unified_provider import get_unified_provider + + provider = get_unified_provider() + + if "trending" in workflow_name.lower(): + trending = await provider.get_dexscreener_trending() + collected_data["data"]["dexscreener_trending"] = trending + elif "volume" in workflow_name.lower(): + spikes = await provider.get_volume_spikes() + collected_data["data"]["dexscreener_spikes"] = spikes + + elif source == "helius": + from app.chain_client import get_chain_client + + client = get_chain_client() + + if "new" in workflow_name.lower(): + new_tokens = await client.get_new_token_mints(limit=50) + collected_data["data"]["helius_new_tokens"] = new_tokens + elif "whale" in workflow_name.lower(): + whales = await client.get_large_transfers(limit=20) + collected_data["data"]["helius_whales"] = whales + + except Exception as e: + logger.error(f"Error collecting from {source}: {e}") + collected_data["data"][source] = {"error": str(e)} + + # Store in Supabase + await _store_intelligence_data(workflow["output_table"], collected_data) + + # Trigger alerts if needed + await _process_intelligence_alerts(workflow_name, collected_data) + + +async def _store_intelligence_data(table_name: str, data: dict): + """Store intelligence data in Supabase.""" + try: + import os + + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + # Insert data + supabase.table(table_name).insert( + { + "data": data, + "collected_at": data.get("collected_at"), + "workflow": data.get("workflow"), + } + ).execute() + + logger.info(f"Stored intelligence data in {table_name}") + + except Exception as e: + logger.error(f"Failed to store intelligence data: {e}") + + +async def _process_intelligence_alerts(workflow_name: str, data: dict): + """Process intelligence data and trigger alerts.""" + # Check for significant findings + if "whale" in workflow_name.lower(): + # Check for unusually large transfers + whales = data.get("data", {}).get("helius_whales", []) + for whale in whales: + amount = whale.get("amount", 0) + if amount > 1000: # >1000 SOL + await _create_alert("whale_movement", whale) + + elif "scam" in workflow_name.lower() or "rugpull" in workflow_name.lower(): + # Immediate alert for security issues + await _create_alert("security_critical", data) + + elif "volume" in workflow_name.lower(): + # Check for unusual volume spikes + spikes = data.get("data", {}).get("dexscreener_spikes", []) + for spike in spikes: + if spike.get("volume_change_pct", 0) > 500: # >500% increase + await _create_alert("volume_spike", spike) + + +async def _create_alert(alert_type: str, data: dict): + """Create an intelligence alert.""" + try: + import os + + from supabase import create_client + + supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) + + alert_data = { + "alert_type": alert_type, + "severity": "critical" if "security" in alert_type else "high", + "data": data, + "created_at": datetime.now(UTC).isoformat(), + "is_read": False, + } + + supabase.table("intelligence_alerts").insert(alert_data).execute() + + logger.info(f"Created intelligence alert: {alert_type}") + + except Exception as e: + logger.error(f"Failed to create alert: {e}") + + +# ── Dune Analytics Integration ──────────────────────────────── + + +async def execute_dune_query(query_id: str, params: dict | None = None) -> dict | None: + """Execute a Dune Analytics query.""" + import os + + import httpx + + dune_api_key = os.getenv("DUNE_API_KEY", "") + if not dune_api_key: + logger.warning("DUNE_API_KEY not configured") + return None + + try: + async with httpx.AsyncClient(timeout=60.0) as client: + # Execute query + response = await client.post( + f"https://api.dune.com/api/v1/query/{query_id}/execute", + headers={"X-Dune-API-Key": dune_api_key}, + json=params or {}, + ) + + if response.status_code != 200: + logger.error(f"Dune query failed: {response.status_code}") + return None + + execution_id = response.json().get("execution_id") + + # Wait for results + import asyncio + + for _ in range(10): # Max 10 attempts + await asyncio.sleep(2) + + result_response = await client.get( + f"https://api.dune.com/api/v1/execution/{execution_id}/results", + headers={"X-Dune-API-Key": dune_api_key}, + ) + + if result_response.status_code == 200: + result = result_response.json() + if result.get("state") == "QUERY_STATE_COMPLETED": + return result.get("result", {}).get("rows", []) + + return None + + except Exception as e: + logger.error(f"Dune query error: {e}") + return None + + +# Pre-configured Dune queries for RMI +DUNE_QUERIES = { + "ethereum_whale_transfers": { + "query_id": "1234567", # Replace with actual query ID + "description": "Track large ETH transfers from known whale wallets", + "schedule": "*/15 * * * *", + }, + "defi_protocol_volumes": { + "query_id": "2345678", + "description": "Daily DEX volumes across major protocols", + "schedule": "0 * * * *", + }, + "nft_wash_trading": { + "query_id": "3456789", + "description": "Detect potential NFT wash trading patterns", + "schedule": "0 */6 * * *", + }, + "stablecoin_flows": { + "query_id": "4567890", + "description": "Track USDC/USDT flows to/from exchanges", + "schedule": "*/30 * * * *", + }, + "new_contract_deployments": { + "query_id": "5678901", + "description": "New contract deployments with large funding", + "schedule": "*/10 * * * *", + }, +} diff --git a/app/domains/intelligence/news_intelligence.py b/app/domains/intelligence/news_intelligence.py new file mode 100644 index 0000000..2ef6565 --- /dev/null +++ b/app/domains/intelligence/news_intelligence.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +""" +RMI News Intelligence v3 - Industry Best +========================================= +AI-powered news pipeline: categorization, sentiment, trending, briefing. +Uses MiniMax ($20/mo flat) + Ollama Cloud. +""" + +import json +import logging +import os +import urllib.request +from collections import Counter +from datetime import UTC, datetime + +logger = logging.getLogger("rmi.news_v3") +OLLAMA_KEY = os.getenv("OLLAMA_API_KEY", os.getenv("DEEPSEEK_API_KEY", "")) +OLLAMA_URL = "https://ollama.com/v1/chat/completions" + +# Simple in-memory trending tracker +_trending_topics = Counter() +_breaking_alerts = [] + + +def analyze_article(title: str, content: str = "") -> dict: + """Full AI analysis of a news article.""" + text = f"{title} {content[:300]}" + + # Category (fast, cached) + from app.ai_pipeline_v3 import classify_news + + category = classify_news(title, content) + + # Sentiment via MiniMax (batched - 1 call per article is fine at flat rate) + sentiment = "neutral" + try: + k = os.getenv("OLLAMA_API_KEY", "") + if not k: + with open("/app/.env") as f: + for line in f: + if line.startswith("OLLAMA_API_KEY"): + k = line.strip().split("=", 1)[1] + break + if not k: + return {"category": category, "sentiment": "neutral", "is_breaking": False} + body = json.dumps( + { + "model": "deepseek-v4-flash", + "messages": [ + { + "role": "system", + "content": "Classify sentiment: BULLISH BEARISH NEUTRAL. Reply one word only.", + }, + {"role": "user", "content": text[:400]}, + ], + "max_tokens": 10, + "temperature": 0.1, + } + ).encode() + req = urllib.request.Request( + OLLAMA_URL, + data=body, + headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, + ) + resp = urllib.request.urlopen(req, timeout=8) + sentiment = json.loads(resp.read())["choices"][0]["message"]["content"].strip().upper() + if "BULL" in sentiment: + sentiment = "bullish" + elif "BEAR" in sentiment: + sentiment = "bearish" + else: + sentiment = "neutral" + except Exception as e: + logger.warning(f"Sentiment failed: {e}") + + # Track trending topics + for word in title.lower().split(): + if len(word) > 4 and word not in ( + "after", + "before", + "while", + "could", + "would", + "should", + "their", + "there", + "these", + "those", + "about", + "which", + ): + _trending_topics[word] += 1 + + # Detect breaking news + is_breaking = category == "SCAM" or any( + w in text.lower() for w in ["hacked", "exploited", "drained", "rug pulled", "emergency"] + ) + + return { + "category": category, + "sentiment": sentiment, + "is_breaking": is_breaking, + "analyzed_at": datetime.now(UTC).isoformat(), + } + + +def get_trending(limit: int = 10) -> list: + """Get trending topics from recent article analysis.""" + return [{"topic": word, "count": count} for word, count in _trending_topics.most_common(limit)] + + +def get_breaking() -> list: + """Get breaking news alerts.""" + return _breaking_alerts[-10:] + + +def daily_briefing() -> str: + """Generate an AI-powered daily news briefing using Ollama Cloud.""" + trending = get_trending(5) + topics = ", ".join(f"{t['topic']}({t['count']})" for t in trending) + + k = OLLAMA_KEY + body = json.dumps( + { + "model": "deepseek-v4-flash", + "messages": [ + { + "role": "system", + "content": "Write a 3-sentence daily crypto news briefing. Mention trending topics. Professional tone. Under 100 words.", + }, + {"role": "user", "content": f"Trending topics: {topics}"}, + ], + "max_tokens": 150, + "temperature": 0.5, + } + ).encode() + + try: + req = urllib.request.Request( + OLLAMA_URL, + data=body, + headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, + ) + resp = urllib.request.urlopen(req, timeout=15) + return json.loads(resp.read())["choices"][0]["message"]["content"].strip() + except Exception: + return f"Daily briefing: {topics} are trending in crypto news today." + + +def news_search(query: str, articles: list, limit: int = 10) -> list: + """Smart search across articles with relevance ranking.""" + results = [] + q_lower = query.lower() + for a in articles: + score = 0 + if q_lower in a.get("title", "").lower(): + score += 10 + if q_lower in a.get("content", "").lower(): + score += 5 + if q_lower in a.get("category", "").lower(): + score += 3 + if score > 0: + a["relevance"] = score + results.append(a) + return sorted(results, key=lambda x: x.get("relevance", 0), reverse=True)[:limit] diff --git a/app/domains/markets/__init__.py b/app/domains/markets/__init__.py new file mode 100644 index 0000000..eb9e0d0 --- /dev/null +++ b/app/domains/markets/__init__.py @@ -0,0 +1,19 @@ +"""Markets domain - public API. + +Phase 4 domain consolidation: app.prediction_market_service -> app.domains.markets. +""" +from __future__ import annotations + +from app.domains.markets.core import ( + PredictionDigest, + PredictionMarket, + PredictionMarketService, + get_prediction_market_service, +) + +__all__ = [ + "PredictionDigest", + "PredictionMarket", + "PredictionMarketService", + "get_prediction_market_service", +] diff --git a/app/domains/markets/core.py b/app/domains/markets/core.py new file mode 100644 index 0000000..0588baf --- /dev/null +++ b/app/domains/markets/core.py @@ -0,0 +1,1122 @@ +""" +RMI Prediction Market Intelligence Service +=========================================== +Multi-source prediction market data aggregation for crypto security intelligence. + +Data sources (all free, zero auth for read-only): + - Polymarket: Gamma API (search/discovery), CLOB API (prices/history), Data API (trades) + - Kalshi: REST API /series, /markets, /events, /orderbook (unauthenticated) + - Limitless: REST API /markets (Base chain, crypto-native) + - Manifold: REST API /markets (play-money, open-source sentiment signals) + +Open-source reference implementations (GitHub): + - homerun (braedonsaunders/homerun): Open-source prediction market platform for + Polymarket + Kalshi. Python strategies, backtesting, data sources, live trading. + - prediction-market-edge-bot: SX Bet + Polymarket aggregator with smart order routing. + - Awesome-Prediction-Market-Tools (aarora4): Curated directory of 50+ tools + including Oddpool (cross-venue aggregator), analytics dashboards, trading bots. + +Architecture: + - Direct external API calls (NEVER route through own API - anti-circular-dependency rule) + - All 4 sources queried in parallel with individual try/except + - Results normalized into unified PredictionMarket dataclass + - Redis caching: 30s TTL prices, 5min searches, 1hr digests + +Integration points: + - SENTINEL scanner: cross-reference token risk scores with market probability + - Wallet Memory Bank: entity/deployer reputation from prediction market odds + - RugMaps: visual correlation between market odds and on-chain wallet clusters + - x402 tools: expose as paid security intelligence endpoints + +Pitfalls: + - Polymarket Gamma API double-encodes outcomePrices/clobTokenIds as JSON strings + - One source timeout must not kill the entire call - individual try/except per source + - Prediction market data is probabilistic, not definitive - always cross-reference + with on-chain scanner results + - Don't poll every market every tick - use targeted search + category filters +""" + +import asyncio +import hashlib +import json +import logging +import os +from dataclasses import dataclass, field +from datetime import UTC, datetime + +import httpx + +logger = logging.getLogger("prediction_market") + +# ── API Endpoints ──────────────────────────────────────────────── + +POLYMARKET_GAMMA = "https://gamma-api.polymarket.com" +POLYMARKET_CLOB = "https://clob.polymarket.com" +POLYMARKET_DATA = "https://data-api.polymarket.com" + +KALSHI_BASE = "https://external-api.kalshi.com/trade-api/v2" + +LIMITLESS_BASE = "https://api.limitless.exchange" + +MANIFOLD_BASE = "https://api.manifold.markets/v0" + +# ── Category mappings for security relevance ──────────────────── + +SECURITY_KEYWORDS = [ + "hack", + "exploit", + "rug", + "scam", + "fraud", + "breach", + "leak", + "drain", + "phish", + "backdoor", + "vulnerability", + "zero-day", + "sanction", + "indict", + "arrest", + "freeze", + "seize", + "clampdown", + "depeg", + "insolvent", + "bankrupt", + "collapse", + "default", + "theft", + "heist", + "compromise", + "ransomware", + "malware", + "SEC", + "CFTC", + "DOJ", + "FBI", + "regulatory", + "enforcement", +] + +CRYPTO_KEYWORDS = [ + "bitcoin", + "ethereum", + "solana", + "crypto", + "defi", + "token", + "blockchain", + "web3", + "nft", + "stablecoin", + "usdt", + "usdc", + "dai", + "exchange", + "binance", + "coinbase", + "uniswap", + "aave", + "tether", + "circle", + "layer", + "L1", + "L2", + "rollup", + "bridge", + "polygon", + "arbitrum", + "optimism", + "avalanche", + "fantom", + "chainlink", + "makerdao", + "lido", + "eigenlayer", +] + +# ── Dataclasses ───────────────────────────────────────────────── + + +@dataclass +class PredictionMarket: + """Unified prediction market result across all sources.""" + + source: str # "polymarket" | "kalshi" | "limitless" | "manifold" + source_id: str # native ID from source (slug, ticker, etc.) + question: str + slug: str + probability_yes: float # 0.0-1.0 + probability_no: float # 0.0-1.0 + volume_usd: float + liquidity_usd: float = 0.0 + category: str = "" + tags: list[str] = field(default_factory=list) + tokens_mentioned: list[str] = field(default_factory=list) + is_security_relevant: bool = False + is_crypto_relevant: bool = False + url: str = "" + ends_at: str | None = None + updated_at: str = "" + + def __post_init__(self): + """Auto-classify relevance based on keywords in question.""" + q_lower = self.question.lower() + self.is_security_relevant = any(kw in q_lower for kw in SECURITY_KEYWORDS) + self.is_crypto_relevant = any(kw in q_lower for kw in CRYPTO_KEYWORDS) + + +@dataclass +class PredictionDigest: + """Daily intelligence digest of security-relevant prediction markets.""" + + generated_at: str + total_markets_searched: int + security_relevant_count: int + crypto_relevant_count: int + top_threats: list[PredictionMarket] = field(default_factory=list) + token_specific_markets: list[PredictionMarket] = field(default_factory=list) + ecosystem_risk_markets: list[PredictionMarket] = field(default_factory=list) + regulatory_markets: list[PredictionMarket] = field(default_factory=list) + + +# ── Service ───────────────────────────────────────────────────── + + +class PredictionMarketService: + """Multi-source prediction market data with parallel fetching + caching.""" + + def __init__(self, http_client: httpx.AsyncClient | None = None): + self._http = http_client or httpx.AsyncClient(timeout=15.0) + self._redis = None # Lazy init via get_redis() + + def _get_redis(self): + """Lazy Redis connection for caching. Returns None if unavailable.""" + if self._redis is not None: + return self._redis + try: + import redis.asyncio as aioredis + + self._redis = aioredis.from_url( + os.getenv("REDIS_URL", "redis://localhost:6379/0"), + decode_responses=True, + socket_connect_timeout=3, + ) + # Set to False if we couldn't actually connect + if self._redis is None: + self._redis = False + except Exception as e: + logger.warning(f"Redis unavailable, caching disabled: {e}") + self._redis = False + return self._redis if self._redis is not False else None + + # ── Public API ────────────────────────────────────────── + + async def search( + self, + query: str, + categories: list[str] | None = None, + min_volume: float = 0, + security_only: bool = False, + ) -> list[PredictionMarket]: + """Search all prediction market sources in parallel. + + Args: + query: Search term (token name, event, protocol, etc.) + categories: Optional filter by source categories + min_volume: Minimum USD volume to include + security_only: Only return security-relevant markets + """ + # Check Redis cache + cache_key = f"predmkt:search:{_cache_hash(query, categories, min_volume, security_only)}" + redis = self._get_redis() + if redis: + try: + cached = await redis.get(cache_key) + if cached: + markets_data = json.loads(cached) + return [_dict_to_market(d) for d in markets_data] + except Exception: + pass # Cache miss or Redis error - fall through to live query + + # Fire all 4 sources in parallel + results: list[list[PredictionMarket]] = await asyncio.gather( + self._search_polymarket(query), + self._search_kalshi(query), + self._search_limitless(query), + self._search_manifold(query), + return_exceptions=True, + ) + + # Flatten and handle exceptions + all_markets: list[PredictionMarket] = [] + sources = ["polymarket", "kalshi", "limitless", "manifold"] + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.warning(f"{sources[i]} search failed: {result}") + continue + if isinstance(result, list): + all_markets.extend(result) + + # Filter + if min_volume > 0: + all_markets = [m for m in all_markets if m.volume_usd >= min_volume] + if security_only: + all_markets = [m for m in all_markets if m.is_security_relevant] + + # Sort by volume descending + all_markets.sort(key=lambda m: m.volume_usd, reverse=True) + + # Cache (5 min TTL) + if redis: + try: + await redis.setex(cache_key, 300, json.dumps([_market_to_dict(m) for m in all_markets[:50]])) + except Exception as e: + logger.warning(f"Redis cache write failed: {e}") + + return all_markets + + async def token_markets(self, symbol: str) -> list[PredictionMarket]: + """Find all prediction markets mentioning a specific token symbol.""" + results = await self.search(f'"{symbol}" token crypto', security_only=False) + + # Filter to markets actually about this token (mention in question) + symbol_lower = symbol.lower() + token_markets = [m for m in results if symbol_lower in m.question.lower()] + return token_markets + + async def security_digest(self) -> PredictionDigest: + """Generate daily intelligence digest of security-relevant prediction markets. + + Queries crypto + security keywords across all sources, categorizes + results by threat type: top threats, token-specific, ecosystem risk, + regulatory. + """ + cache_key = f"predmkt:digest:{datetime.now(UTC).strftime('%Y-%m-%d')}" + redis = self._get_redis() + if redis: + try: + cached = await redis.get(cache_key) + if cached: + data = json.loads(cached) + return _dict_to_digest(data) + except Exception: + pass # Cache miss or Redis error - fall through to live query + + # Search for security-relevant crypto markets + security_queries = [ + "crypto hack exploit scam", + "defi rug pull fraud", + "exchange insolvent breach", + "stablecoin depeg collapse", + "crypto regulation SEC enforcement", + "blockchain vulnerability zero-day", + ] + + all_markets: list[PredictionMarket] = [] + searches = [self.search(q, security_only=False) for q in security_queries] + results = await asyncio.gather(*searches, return_exceptions=True) + + for result in results: + if isinstance(result, list): + all_markets.extend(result) + + # De-duplicate by question similarity + seen_questions = set() + unique_markets = [] + for m in all_markets: + q_key = m.question.lower().strip()[:80] + if q_key not in seen_questions: + seen_questions.add(q_key) + unique_markets.append(m) + + # Categorize + top_threats = [m for m in unique_markets if m.is_security_relevant and m.volume_usd > 10000] + top_threats.sort(key=lambda m: m.volume_usd, reverse=True) + + token_specific = [ + m for m in unique_markets if m.is_crypto_relevant and m.is_security_relevant and len(m.tokens_mentioned) > 0 + ] + token_specific.sort(key=lambda m: m.volume_usd, reverse=True) + + ecosystem_risk = [ + m for m in unique_markets if m.is_crypto_relevant and not m.is_security_relevant and m.volume_usd > 50000 + ] + ecosystem_risk.sort(key=lambda m: m.volume_usd, reverse=True) + + regulatory = [ + m + for m in unique_markets + if m.is_security_relevant + and any(kw in m.question.lower() for kw in ["sec", "cftc", "doj", "regulation", "sanction", "ban"]) + ] + regulatory.sort(key=lambda m: m.volume_usd, reverse=True) + + digest = PredictionDigest( + generated_at=datetime.now(UTC).isoformat(), + total_markets_searched=len(unique_markets), + security_relevant_count=len([m for m in unique_markets if m.is_security_relevant]), + crypto_relevant_count=len([m for m in unique_markets if m.is_crypto_relevant]), + top_threats=top_threats[:20], + token_specific_markets=token_specific[:20], + ecosystem_risk_markets=ecosystem_risk[:10], + regulatory_markets=regulatory[:10], + ) + + # Cache (1 hour) + if redis: + try: + await redis.setex(cache_key, 3600, json.dumps(_digest_to_dict(digest))) + except Exception as e: + logger.warning(f"Redis digest cache write failed: {e}") + + return digest + + async def trending(self, limit: int = 20, source: str | None = None) -> list[PredictionMarket]: + """Get top trending prediction markets by volume across all sources.""" + # Fetch top events from each source in parallel + tasks = [] + if not source or source == "polymarket": + tasks.append(self._trending_polymarket(limit)) + else: + tasks.append(asyncio.sleep(0)) # placeholder + + if not source or source == "kalshi": + tasks.append(self._trending_kalshi(limit)) + else: + tasks.append(asyncio.sleep(0)) + + if not source or source == "limitless": + tasks.append(self._trending_limitless(limit)) + else: + tasks.append(asyncio.sleep(0)) + + results = await asyncio.gather(*tasks, return_exceptions=True) + + all_markets = [] + for result in results: + if isinstance(result, list): + all_markets.extend(result) + elif isinstance(result, Exception): + pass # Individual source failures logged in _trending_* methods + + all_markets.sort(key=lambda m: m.volume_usd, reverse=True) + return all_markets[:limit] + + async def market_detail(self, source: str, market_id: str) -> PredictionMarket | None: + """Get detailed data for a specific market including orderbook.""" + if source == "polymarket": + return await self._polymarket_detail(market_id) + elif source == "kalshi": + return await self._kalshi_detail(market_id) + # Limitless and Manifold details on demand + return None + + # ── Polymarket ────────────────────────────────────────── + + async def _search_polymarket(self, query: str) -> list[PredictionMarket]: + """Search Polymarket Gamma API.""" + try: + resp = await self._http.get( + f"{POLYMARKET_GAMMA}/public-search", + params={"q": query}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Polymarket search returned {resp.status_code}") + return [] + + data = resp.json() + events = data.get("events", []) + markets = [] + + for event in events[:10]: + for m in event.get("markets", [])[:5]: + pm = self._parse_polymarket_market(m, event) + if pm: + markets.append(pm) + + return markets + except Exception as e: + logger.warning(f"Polymarket search error: {e}") + return [] + + def _parse_polymarket_market(self, m: dict, event: dict | None = None) -> PredictionMarket | None: + """Parse a Polymarket market dict into unified PredictionMarket.""" + try: + question = m.get("question", "") + slug = m.get("slug", "") + + # Parse double-encoded JSON fields + prices = self._parse_json_field(m.get("outcomePrices", "[]")) + self._parse_json_field(m.get("outcomes", "[]")) + self._parse_json_field(m.get("clobTokenIds", "[]")) + + if isinstance(prices, list) and len(prices) >= 2: + prob_yes = float(prices[0]) + prob_no = float(prices[1]) + else: + prob_yes = 0.5 + prob_no = 0.5 + + volume = float(m.get("volume", 0)) + liquidity = float(m.get("liquidity", 0)) + + # Extract token mentions from question + tokens_mentioned = _extract_token_symbols(question) + + tags = [] + if event: + tags.extend([t.get("label", "") for t in event.get("tags", [])]) + + return PredictionMarket( + source="polymarket", + source_id=slug, + question=question, + slug=slug, + probability_yes=prob_yes, + probability_no=prob_no, + volume_usd=volume, + liquidity_usd=liquidity, + category=m.get("category", event.get("category", "") if event else ""), + tags=tags, + tokens_mentioned=tokens_mentioned, + url=f"https://polymarket.com/event/{slug}" if slug else "", + ends_at=m.get("endDate", event.get("endDate", "") if event else ""), + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Polymarket market: {e}") + return None + + async def _trending_polymarket(self, limit: int) -> list[PredictionMarket]: + """Get trending Polymarket events by volume.""" + try: + resp = await self._http.get( + f"{POLYMARKET_GAMMA}/events", + params={ + "limit": limit, + "active": "true", + "closed": "false", + "order": "volume", + "ascending": "false", + }, + timeout=10.0, + ) + if resp.status_code != 200: + return [] + + events = resp.json() + markets = [] + for event in events[:limit]: + for m in event.get("markets", [])[:3]: + pm = self._parse_polymarket_market(m, event) + if pm: + markets.append(pm) + return markets + except Exception as e: + logger.warning(f"Polymarket trending error: {e}") + return [] + + async def _polymarket_detail(self, slug: str) -> PredictionMarket | None: + """Get detailed Polymarket market data including CLOB prices.""" + try: + # Fetch from Gamma + resp = await self._http.get( + f"{POLYMARKET_GAMMA}/markets", + params={"slug": slug}, + timeout=10.0, + ) + if resp.status_code != 200: + return None + + data = resp.json() + if not data: + return None + + m = data[0] + pm = self._parse_polymarket_market(m) + + # Also fetch CLOB price for live data + if pm: + tokens = self._parse_json_field(m.get("clobTokenIds", "[]")) + if isinstance(tokens, list) and len(tokens) >= 2: + try: + price_resp = await self._http.get( + f"{POLYMARKET_CLOB}/price", + params={"token_id": tokens[0], "side": "buy"}, + timeout=5.0, + ) + if price_resp.status_code == 200: + price_data = price_resp.json() + live_price = float(price_data.get("price", pm.probability_yes)) + pm.probability_yes = live_price + pm.probability_no = 1.0 - live_price + except Exception: + pass # CLOB price is a bonus, Gamma price is fine + + return pm + except Exception as e: + logger.warning(f"Polymarket detail error for {slug}: {e}") + return None + + # ── Kalshi ────────────────────────────────────────────── + + async def _search_kalshi(self, query: str) -> list[PredictionMarket]: + """Search Kalshi by scanning events then fetching their markets.""" + try: + # Step 1: Get open events (organized by category, not sports-dominant) + resp = await self._http.get( + f"{KALSHI_BASE}/events", + params={"status": "open", "limit": 50}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Kalshi events returned {resp.status_code}") + return [] + + data = resp.json() + events = data.get("events", []) + query_lower = query.lower() + query_terms = query_lower.split() + + results = [] + + # Step 2: Check event titles for matches, then fetch markets + for i, event in enumerate(events[:10]): + event_title = event.get("title", "").lower() + event_ticker = event.get("ticker", "") + + # Match if query terms appear in event title + if not any(term in event_title for term in query_terms): + continue + + # Rate limit: small delay between event fetches + if i > 0: + await asyncio.sleep(0.3) + + # Step 3: Fetch markets for this event + try: + mr = await self._http.get( + f"{KALSHI_BASE}/markets", + params={"event_ticker": event_ticker, "status": "open", "limit": 10}, + headers={"Accept": "application/json"}, + timeout=8.0, + ) + if mr.status_code == 200: + markets_data = mr.json() + for m in markets_data.get("markets", []): + pm = self._parse_kalshi_market(m) + if pm: + results.append(pm) + except Exception: + continue + + return results + except Exception as e: + logger.warning(f"Kalshi search error: {e}") + return [] + + def _parse_kalshi_market(self, m: dict) -> PredictionMarket | None: + """Parse a Kalshi market dict into unified PredictionMarket.""" + try: + ticker = m.get("ticker", "") + title = m.get("title", "") + yes_bid = float(m.get("yes_bid_dollars", 0)) + volume = float(m.get("volume_fp", 0)) # Kalshi uses fake-penny notation + m.get("event_ticker", "") + category = m.get("category", "") + + # Skip multi-outcome markets (sports parlays, etc.) - they have no yes_bid + if yes_bid <= 0 or ",yes " in title.lower(): + return None + + prob_yes = yes_bid # Best YES bid approximates probability + prob_no = 1.0 - prob_yes if prob_yes else 0.5 + + tokens_mentioned = _extract_token_symbols(title) + + return PredictionMarket( + source="kalshi", + source_id=ticker, + question=title, + slug=ticker, + probability_yes=prob_yes, + probability_no=prob_no, + volume_usd=volume, + category=category, + tokens_mentioned=tokens_mentioned, + url=f"https://kalshi.com/markets/{ticker}" if ticker else "", + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Kalshi market: {e}") + return None + + async def _trending_kalshi(self, limit: int) -> list[PredictionMarket]: + """Get trending Kalshi markets by volume - uses events-first approach.""" + try: + # Get open events (avoid sports-multi-outcome noise from raw /markets) + resp = await self._http.get( + f"{KALSHI_BASE}/events", + params={"status": "open", "limit": min(limit * 2, 30)}, + timeout=10.0, + ) + if resp.status_code != 200: + return [] + + data = resp.json() + events = data.get("events", []) + + results = [] + for event in events[:limit]: + event_ticker = event.get("ticker", "") + try: + mr = await self._http.get( + f"{KALSHI_BASE}/markets", + params={"event_ticker": event_ticker, "status": "open", "limit": 5}, + timeout=8.0, + ) + if mr.status_code == 200: + markets_data = mr.json() + for m in markets_data.get("markets", []): + pm = self._parse_kalshi_market(m) + if pm: + results.append(pm) + except Exception: + continue + + return results[:limit] + except Exception as e: + logger.warning(f"Kalshi trending error: {e}") + return [] + + async def _kalshi_detail(self, ticker: str) -> PredictionMarket | None: + """Get detailed Kalshi market data including orderbook.""" + try: + resp = await self._http.get( + f"{KALSHI_BASE}/markets/{ticker}/orderbook", + timeout=10.0, + ) + if resp.status_code != 200: + return None + + data = resp.json() + orderbook = data.get("orderbook_fp", {}) + yes_bids = orderbook.get("yes_dollars", []) + best_yes = float(yes_bids[0][0]) if yes_bids else 0.5 + + # Also get market metadata + meta_resp = await self._http.get( + f"{KALSHI_BASE}/markets", + params={"ticker": ticker}, + timeout=10.0, + ) + if meta_resp.status_code == 200: + meta_data = meta_resp.json() + markets = meta_data.get("markets", []) + if markets: + pm = self._parse_kalshi_market(markets[0]) + if pm: + pm.probability_yes = best_yes + pm.probability_no = 1.0 - best_yes + return pm + + return None + except Exception as e: + logger.warning(f"Kalshi detail error for {ticker}: {e}") + return None + + # ── Limitless ─────────────────────────────────────────── + + async def _search_limitless(self, query: str) -> list[PredictionMarket]: + """Search Limitless Exchange markets by fetching active and filtering.""" + try: + resp = await self._http.get( + f"{LIMITLESS_BASE}/markets/active", + params={"limit": 25}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Limitless markets returned {resp.status_code}") + return [] + + data = resp.json() + all_markets = data.get("data", []) + + # Filter client-side by query terms + query_lower = query.lower() + query_terms = query_lower.split() + + results = [] + for m in all_markets: + title = m.get("title", "").lower() + if any(term in title for term in query_terms): + pm = self._parse_limitless_market(m) + if pm: + results.append(pm) + + return results + except Exception as e: + logger.warning(f"Limitless search error: {e}") + return [] + + def _parse_limitless_market(self, m: dict) -> PredictionMarket | None: + """Parse a Limitless market dict into unified PredictionMarket.""" + try: + title = m.get("title", "") + slug = m.get("slug", str(m.get("id", ""))) + + # Prices: [YES%, NO%] - e.g., [42.8, 57.2] + prices = m.get("prices", [50, 50]) + prob_yes = float(prices[0]) / 100 if isinstance(prices, list) and len(prices) >= 1 else 0.5 + prob_no = float(prices[1]) / 100 if isinstance(prices, list) and len(prices) >= 2 else 0.5 + + # Volume: use volumeFormatted if available, else volume + vol_str = m.get("volumeFormatted", str(m.get("volume", 0))) + volume = float(vol_str) if vol_str else 0.0 + + categories = m.get("categories", []) + category = categories[0] if categories else "" + tags = m.get("tags", []) + tokens_mentioned = _extract_token_symbols(title) + + return PredictionMarket( + source="limitless", + source_id=str(slug), + question=title, + slug=str(slug), + probability_yes=prob_yes, + probability_no=prob_no, + volume_usd=volume, + category=category, + tags=tags, + tokens_mentioned=tokens_mentioned, + url=f"https://limitless.exchange/markets/{slug}" if slug else "", + ends_at=m.get("expirationDate", ""), + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Limitless market: {e}") + return None + + async def _trending_limitless(self, limit: int) -> list[PredictionMarket]: + """Get trending Limitless markets.""" + try: + limit = min(limit, 25) # API max + resp = await self._http.get( + f"{LIMITLESS_BASE}/markets/active", + params={"limit": limit}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + if resp.status_code != 200: + return [] + + data = resp.json() + markets = data.get("data", []) + return [pm for m in markets[:limit] if (pm := self._parse_limitless_market(m))] + except Exception as e: + logger.warning(f"Limitless trending error: {e}") + return [] + + # ── Manifold ──────────────────────────────────────────── + + async def _search_manifold(self, query: str) -> list[PredictionMarket]: + """Search Manifold Markets (play-money, sentiment signals). + + Manifold is pure play-money but useful for: + - Forecasting community sentiment + - Early signal detection (top forecasters often move before real-money markets) + - Broad question coverage (more niche crypto questions than Polymarket) + """ + try: + resp = await self._http.get( + f"{MANIFOLD_BASE}/search-markets", + params={"term": query, "limit": 20}, + timeout=10.0, + ) + if resp.status_code != 200: + logger.warning(f"Manifold search returned {resp.status_code}") + return [] + + data = resp.json() + contracts = data if isinstance(data, list) else data.get("contracts", data.get("markets", [])) + + results = [] + for c in contracts[:10]: + pm = self._parse_manifold_market(c) + if pm: + results.append(pm) + + return results + except Exception as e: + logger.warning(f"Manifold search error: {e}") + return [] + + def _parse_manifold_market(self, c: dict) -> PredictionMarket | None: + """Parse a Manifold contract into unified PredictionMarket.""" + try: + question = c.get("question", "") + slug = c.get("slug", c.get("id", "")) + prob = float(c.get("probability", c.get("prob", 0.5))) + volume = float(c.get("volume", c.get("volume24Hours", 0))) + + # Manifold uses "Mana" play money, volume is a signal but lower weight + tokens_mentioned = _extract_token_symbols(question) + tags = list(c.get("tags", [])) + + return PredictionMarket( + source="manifold", + source_id=str(slug), + question=question, + slug=str(slug), + probability_yes=prob, + probability_no=1.0 - prob, + volume_usd=volume, + category="", + tags=tags, + tokens_mentioned=tokens_mentioned, + url=f"https://manifold.markets/{c.get('creatorUsername', '')}/{slug}" if slug else "", + updated_at=datetime.now(UTC).isoformat(), + ) + except Exception as e: + logger.warning(f"Failed to parse Manifold market: {e}") + return None + + # ── Helpers ───────────────────────────────────────────── + + @staticmethod + def _parse_json_field(val): + """Parse double-encoded JSON fields (Polymarket Gamma API).""" + if isinstance(val, str): + try: + return json.loads(val) + except (json.JSONDecodeError, TypeError): + return val + return val + + +# ── Singleton ──────────────────────────────────────────────────── + +_service: PredictionMarketService | None = None + + +def get_prediction_market_service() -> PredictionMarketService: + """Get or create the singleton PredictionMarketService.""" + global _service + if _service is None: + _service = PredictionMarketService() + return _service + + +# ── Helpers: Token Extraction ──────────────────────────────────── + +# Common token symbols to detect in market questions +_COMMON_TOKENS = { + "BTC", + "ETH", + "SOL", + "USDT", + "USDC", + "DAI", + "BNB", + "XRP", + "ADA", + "DOGE", + "MATIC", + "POL", + "DOT", + "AVAX", + "LINK", + "UNI", + "AAVE", + "ARB", + "OP", + "SUI", + "APT", + "TIA", + "SEI", + "STRK", + "WLD", + "PEPE", + "SHIB", + "BONK", + "WIF", + "JUP", + "PYTH", + "RNDR", + "FET", + "AGIX", + "OCEAN", + "IMX", + "INJ", + "EIGEN", + "ENA", + "ETHFI", +} + +# Broader project names often referenced in prediction markets +_COMMON_PROJECTS = { + "polymarket", + "kalshi", + "manifold", + "uniswap", + "sushiswap", + "aave", + "compound", + "makerdao", + "maker", + "lido", + "eigenlayer", + "chainlink", + "arbitrum", + "optimism", + "polygon", + "avalanche", + "fantom", + "near", + "celestia", + "worldcoin", + "tether", + "circle", + "coinbase", + "binance", + "kraken", + "ftx", + "celcius", + "blockfi", + "three arrows", + "alameda", + "jump crypto", + "wintermute", + "curve", + "balancer", + "thorchain", + "osmosis", + "dydx", + "gmx", + "hyperliquid", + "jupiter", + "raydium", + "orca", + "wormhole", + "layerzero", + "zksync", + "starknet", + "scroll", + "linea", + "base", + "mantle", + "mode", + "blast", +} + + +def _extract_token_symbols(text: str) -> list[str]: + """Extract known token symbols and project names from text.""" + found = [] + text_upper = text.upper() + text_lower = text.lower() + + # Check token symbols (typically uppercase in text) + for token in _COMMON_TOKENS: + # Match as word boundary: " BTC " or "BTC's" or "$BTC" + if ( + f" {token} " in f" {text_upper} " + or f"${token}" in text_upper + or text_upper.startswith(f"{token} ") + or text_upper.endswith(f" {token}") + ) and token not in found: + found.append(token) + + # Check project names (case-insensitive) + for project in _COMMON_PROJECTS: + if project in text_lower and project.upper() not in found: + found.append(project) + + return found + + +# ── Caching Helpers ────────────────────────────────────────────── + + +def _cache_hash(*args) -> str: + """Create a short hash for cache keys.""" + raw = "|".join(str(a) for a in args) + return hashlib.md5(raw.encode()).hexdigest()[:12] + + +def _market_to_dict(m: PredictionMarket) -> dict: + """Serialize PredictionMarket to dict for JSON caching.""" + return { + "source": m.source, + "source_id": m.source_id, + "question": m.question, + "slug": m.slug, + "probability_yes": m.probability_yes, + "probability_no": m.probability_no, + "volume_usd": m.volume_usd, + "liquidity_usd": m.liquidity_usd, + "category": m.category, + "tags": m.tags, + "tokens_mentioned": m.tokens_mentioned, + "is_security_relevant": m.is_security_relevant, + "is_crypto_relevant": m.is_crypto_relevant, + "url": m.url, + "ends_at": m.ends_at, + "updated_at": m.updated_at, + } + + +def _dict_to_market(d: dict) -> PredictionMarket: + """Deserialize dict back to PredictionMarket.""" + return PredictionMarket( + source=d.get("source", ""), + source_id=d.get("source_id", ""), + question=d.get("question", ""), + slug=d.get("slug", ""), + probability_yes=float(d.get("probability_yes", 0.5)), + probability_no=float(d.get("probability_no", 0.5)), + volume_usd=float(d.get("volume_usd", 0)), + liquidity_usd=float(d.get("liquidity_usd", 0)), + category=d.get("category", ""), + tags=d.get("tags", []), + tokens_mentioned=d.get("tokens_mentioned", []), + is_security_relevant=d.get("is_security_relevant", False), + is_crypto_relevant=d.get("is_crypto_relevant", False), + url=d.get("url", ""), + ends_at=d.get("ends_at", ""), + updated_at=d.get("updated_at", ""), + ) + + +def _digest_to_dict(d: PredictionDigest) -> dict: + """Serialize PredictionDigest to dict for JSON caching.""" + return { + "generated_at": d.generated_at, + "total_markets_searched": d.total_markets_searched, + "security_relevant_count": d.security_relevant_count, + "crypto_relevant_count": d.crypto_relevant_count, + "top_threats": [_market_to_dict(m) for m in d.top_threats], + "token_specific_markets": [_market_to_dict(m) for m in d.token_specific_markets], + "ecosystem_risk_markets": [_market_to_dict(m) for m in d.ecosystem_risk_markets], + "regulatory_markets": [_market_to_dict(m) for m in d.regulatory_markets], + } + + +def _dict_to_digest(d: dict) -> PredictionDigest: + """Deserialize dict back to PredictionDigest.""" + return PredictionDigest( + generated_at=d.get("generated_at", ""), + total_markets_searched=d.get("total_markets_searched", 0), + security_relevant_count=d.get("security_relevant_count", 0), + crypto_relevant_count=d.get("crypto_relevant_count", 0), + top_threats=[_dict_to_market(m) for m in d.get("top_threats", [])], + token_specific_markets=[_dict_to_market(m) for m in d.get("token_specific_markets", [])], + ecosystem_risk_markets=[_dict_to_market(m) for m in d.get("ecosystem_risk_markets", [])], + regulatory_markets=[_dict_to_market(m) for m in d.get("regulatory_markets", [])], + ) diff --git a/app/domains/mcp/__init__.py b/app/domains/mcp/__init__.py new file mode 100644 index 0000000..458132e --- /dev/null +++ b/app/domains/mcp/__init__.py @@ -0,0 +1,33 @@ +"""MCP domain - public API. + +Phase 4 domain consolidation: app.mcp -> app.domains.mcp. +""" +from __future__ import annotations + +from app.domains.mcp.registry import TOOL_CATEGORIES, get_tool_by_name +from app.domains.mcp.server import ( + MCP_PROTOCOL_VERSION, + MCP_SERVER_VERSION, + TOOL_CATALOG, + TOOL_DEPRECATED, + TOOL_SUCCESSORS, + TOOL_VERSIONS, + call_tool, +) +from app.domains.mcp.x402_mcp_server import TOOLS, handle_mcp_call +from app.domains.mcp.x402_tool_manager import X402ToolManager + +__all__ = [ + "MCP_PROTOCOL_VERSION", + "MCP_SERVER_VERSION", + "TOOLS", + "TOOL_CATALOG", + "TOOL_CATEGORIES", + "TOOL_DEPRECATED", + "TOOL_SUCCESSORS", + "TOOL_VERSIONS", + "X402ToolManager", + "call_tool", + "get_tool_by_name", + "handle_mcp_call", +] diff --git a/app/mcp/manifest.py b/app/domains/mcp/manifest.py similarity index 100% rename from app/mcp/manifest.py rename to app/domains/mcp/manifest.py diff --git a/app/mcp/registry.py b/app/domains/mcp/registry.py similarity index 98% rename from app/mcp/registry.py rename to app/domains/mcp/registry.py index 8f60fc3..0b5330b 100644 --- a/app/mcp/registry.py +++ b/app/domains/mcp/registry.py @@ -4,8 +4,8 @@ from __future__ import annotations import logging -from app.mcp.manifest import MCPServerManifest, MCPToolManifest -from app.mcp.server import ( +from app.domains.mcp.manifest import MCPServerManifest, MCPToolManifest +from app.domains.mcp.server import ( TOOL_CATALOG, TOOL_DEPRECATED, TOOL_SUCCESSORS, diff --git a/app/api/v1/mcp/router.py b/app/domains/mcp/router.py similarity index 99% rename from app/api/v1/mcp/router.py rename to app/domains/mcp/router.py index 9e95279..c230990 100644 --- a/app/api/v1/mcp/router.py +++ b/app/domains/mcp/router.py @@ -18,7 +18,7 @@ from typing import Any from fastapi import APIRouter, Request from pydantic import BaseModel -from app.mcp.server import ( +from app.domains.mcp.server import ( MCP_PROTOCOL_VERSION, MCP_SERVER_VERSION, TOOL_CATALOG, diff --git a/app/mcp/server.py b/app/domains/mcp/server.py similarity index 99% rename from app/mcp/server.py rename to app/domains/mcp/server.py index 3c165c7..5199653 100644 --- a/app/mcp/server.py +++ b/app/domains/mcp/server.py @@ -601,7 +601,7 @@ async def call_tool(name: str, arguments: dict) -> dict: limit = min(int(arguments.get("limit", 1000)), 10000) try: - from app.mcp.tools.eth_labels_tool import query_eth_labels_db_mcp + from app.domains.mcp.tools.eth_labels_tool import query_eth_labels_db_mcp result = await query_eth_labels_db_mcp(sql, limit) return {"result": result} @@ -611,7 +611,7 @@ async def call_tool(name: str, arguments: dict) -> dict: if name == "eth_labels_stats": # T22: Get statistics about eth-labels.db try: - from app.mcp.tools.eth_labels_tool import get_eth_labels_stats_mcp + from app.domains.mcp.tools.eth_labels_tool import get_eth_labels_stats_mcp result = await get_eth_labels_stats_mcp() return {"result": result} @@ -620,7 +620,7 @@ async def call_tool(name: str, arguments: dict) -> dict: if name == "mcp_discover": # MCP catalog discovery with versioning + deprecation + category filtering - from app.mcp.registry import TOOL_CATEGORIES # lazy: avoid circular import + from app.domains.mcp.registry import TOOL_CATEGORIES # lazy: avoid circular import include_deprecated = arguments.get("include_deprecated", False) category = arguments.get("category") diff --git a/app/mcp/tools/eth_labels_tool.py b/app/domains/mcp/tools/eth_labels_tool.py similarity index 100% rename from app/mcp/tools/eth_labels_tool.py rename to app/domains/mcp/tools/eth_labels_tool.py diff --git a/app/mcp/x402_ai_guard.py b/app/domains/mcp/x402_ai_guard.py similarity index 100% rename from app/mcp/x402_ai_guard.py rename to app/domains/mcp/x402_ai_guard.py diff --git a/app/mcp/x402_bundles.py b/app/domains/mcp/x402_bundles.py similarity index 100% rename from app/mcp/x402_bundles.py rename to app/domains/mcp/x402_bundles.py diff --git a/app/mcp/x402_mcp_server.py b/app/domains/mcp/x402_mcp_server.py similarity index 100% rename from app/mcp/x402_mcp_server.py rename to app/domains/mcp/x402_mcp_server.py diff --git a/app/mcp/x402_tool_manager.py b/app/domains/mcp/x402_tool_manager.py similarity index 100% rename from app/mcp/x402_tool_manager.py rename to app/domains/mcp/x402_tool_manager.py diff --git a/app/domains/referral/__init__.py b/app/domains/referral/__init__.py new file mode 100644 index 0000000..5bd3cda --- /dev/null +++ b/app/domains/referral/__init__.py @@ -0,0 +1,31 @@ +"""Referral domain - public API. + +Phase 4 domain consolidation: extracted from telegram/rugmunchbot/bot.py. +""" +from __future__ import annotations + +from app.domains.referral.core import ( + MILESTONES, + REFERRAL_BONUS_SCANS, + apply_referral_code, + generate_referral_link, + get_referral_stats, +) +from app.domains.referral.partners import ( + DEX_REF_LINKS, + PARTNER_CODES, + dex_buttons, + format_dex_url, +) + +__all__ = [ + "DEX_REF_LINKS", + "MILESTONES", + "PARTNER_CODES", + "REFERRAL_BONUS_SCANS", + "apply_referral_code", + "dex_buttons", + "format_dex_url", + "generate_referral_link", + "get_referral_stats", +] diff --git a/app/domains/referral/core.py b/app/domains/referral/core.py new file mode 100644 index 0000000..9f115d5 --- /dev/null +++ b/app/domains/referral/core.py @@ -0,0 +1,77 @@ +"""Referral domain - Telegram bot referral system. + +Phase 4 domain consolidation: extracted from telegram/rugmunchbot/bot.py. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import sqlite3 + +REFERRAL_BONUS_SCANS = 5 + +MILESTONES = {5: "🥉", 10: "🥈", 25: "🥇", 50: "💎", 100: "👑"} + + +def apply_referral_code( + user_id: int, + ref_code: str, + user_referral_code: str | None, + referred_by: str | None, + conn: sqlite3.Connection, +) -> bool: + """Apply a referral code for a new/existing user. + + Returns True if the referral was successfully applied. + """ + if not ref_code or ref_code == user_referral_code or referred_by: + return False + + referrer = conn.execute( + "SELECT user_id FROM users WHERE referral_code = ?", (ref_code,) + ).fetchone() + if not referrer: + return False + + conn.execute( + "UPDATE users SET referred_by = ?, bonus_scans = bonus_scans + ? WHERE user_id = ?", + (ref_code, REFERRAL_BONUS_SCANS, user_id), + ) + conn.execute( + "UPDATE users SET bonus_scans = bonus_scans + ? WHERE user_id = ?", + (REFERRAL_BONUS_SCANS, referrer["user_id"]), + ) + conn.commit() + return True + + +def get_referral_stats(user_id: int, referral_code: str, conn: sqlite3.Connection) -> dict[str, Any]: + """Return referral stats for a user. + + Includes total referrals, bonus scans earned, next milestone info, and share link. + """ + count_row = conn.execute( + "SELECT COUNT(*) as c FROM users WHERE referred_by = ?", (referral_code,) + ).fetchone() + count = count_row["c"] if count_row else 0 + + next_ms = next((k for k in sorted(MILESTONES.keys()) if k > count), None) + ms_text = ( + f"\n🎯 Next milestone: {MILESTONES[next_ms]} {next_ms} referrals" + if next_ms + else "\n👑 You've hit all milestones!" + ) + + return { + "user_id": user_id, + "referral_code": referral_code, + "count": count, + "bonus_earned": count * REFERRAL_BONUS_SCANS, + "next_milestone_text": ms_text, + } + + +def generate_referral_link(bot_username: str, referral_code: str) -> str: + """Generate a Telegram referral start link.""" + return f"https://t.me/{bot_username}?start=ref_{referral_code}" diff --git a/app/domains/referral/partners.py b/app/domains/referral/partners.py new file mode 100644 index 0000000..81349b2 --- /dev/null +++ b/app/domains/referral/partners.py @@ -0,0 +1,93 @@ +"""Referral partner links and helpers. + +Phase 4 domain consolidation: DEX/DeFi partner referral data extracted from +telegram bot config so it can be reused by scanners, web, and Telegram. +""" +from __future__ import annotations + +from telegram import InlineKeyboardButton + +# Partner referral codes / handles. Keep canonical here. +PARTNER_CODES = { + "jupiter": "rugmunch", + "photon": "rugmunch", + "bullx": "rugmunch", + "gmgn": "rugmunch", + "birdeye": "rugmunch", + "oneinch": "rugmunch", + "pancakeswap": "rugmunch", +} + +DEX_REF_LINKS = { + "jupiter": { + "name": "Jupiter", + "emoji": "🪐", + "url": "https://jup.ag/swap/SOL-{token}?ref=rugmunch", + "chains": ["solana"], + "description": "Best Solana DEX aggregator", + }, + "photon": { + "name": "Photon", + "emoji": "⚡", + "url": "https://photon-sol.tinyastro.io/en/lp/{token}?handle=rugmunch", + "chains": ["solana", "ethereum", "base"], + "description": "Fast Solana trading terminal", + }, + "bullx": { + "name": "BullX", + "emoji": "🐂", + "url": "https://neo.bullx.io/terminal?chain={chain}&address={token}&ref=rugmunch", + "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], + "description": "Multi-chain trading bot", + }, + "gmgn": { + "name": "GMGN.ai", + "emoji": "📊", + "url": "https://gmgn.ai/sol/token/{token}?ref=rugmunch", + "chains": ["solana", "ethereum", "base"], + "description": "Smart money tracking + trading", + }, + "birdeye": { + "name": "Birdeye", + "emoji": "🦅", + "url": "https://birdeye.so/token/{token}?ref=rugmunch", + "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], + "description": "Token analytics & charts", + }, + "oneinch": { + "name": "1inch", + "emoji": "🦄", + "url": "https://app.1inch.io/#/{chain_id}/simple/swap/ETH/{token}?ref=rugmunch", + "chains": ["ethereum", "base", "arbitrum", "bsc", "polygon", "avalanche"], + "description": "Best EVM DEX aggregator", + }, + "pancakeswap": { + "name": "PancakeSwap", + "emoji": "🥞", + "url": "https://pancakeswap.finance/swap?inputCurrency=BNB&outputCurrency={token}&ref=rugmunch", + "chains": ["bsc", "ethereum", "arbitrum", "base"], + "description": "BSC's #1 DEX", + }, +} + + +def format_dex_url(platform: str, token: str, chain: str, chain_id: int) -> str | None: + """Return a formatted referral URL for a DEX partner, or None if unsupported.""" + dex = DEX_REF_LINKS.get(platform) + if not dex or chain.lower() not in dex.get("chains", []): + return None + return dex["url"].format(token=token, chain=chain, chain_id=chain_id) + + +def dex_buttons(token: str, chain: str, chain_id: int) -> list[InlineKeyboardButton]: + """Generate Telegram inline buttons for DEX partners supporting the given chain.""" + buttons = [] + for _key, dex in DEX_REF_LINKS.items(): + if chain.lower() in dex.get("chains", []): + url = dex["url"].format( + token=token, + chain=chain, + chain_id=chain_id, + ) + buttons.append(InlineKeyboardButton(f"{dex['emoji']} {dex['name']}", url=url)) + return buttons diff --git a/app/domains/telegram/rugmunchbot/bot.py b/app/domains/telegram/rugmunchbot/bot.py index fc4cc3c..41c26a2 100644 --- a/app/domains/telegram/rugmunchbot/bot.py +++ b/app/domains/telegram/rugmunchbot/bot.py @@ -42,6 +42,12 @@ from telegram.ext import ( ) # ── Local imports ── +from app.domains.referral import ( + apply_referral_code, + dex_buttons, + generate_referral_link, + get_referral_stats, +) from app.domains.telegram.rugmunchbot import db from app.domains.telegram.rugmunchbot.config import ( BACKEND_URL, @@ -51,7 +57,6 @@ from app.domains.telegram.rugmunchbot.config import ( BOT_USERNAME, CHAIN_IDS, CHANNELS, - DEX_REF_LINKS, DEXSCREENER, GOPLUS, HONEYPOT, @@ -157,18 +162,6 @@ def web_scan_button(address: str, chain: str = "") -> InlineKeyboardButton: return InlineKeyboardButton("🌐 Full Report on RugMunch.io", url=url) -def dex_buttons(token: str, chain: str) -> list[InlineKeyboardButton]: - buttons = [] - for _key, dex in DEX_REF_LINKS.items(): - if chain.lower() in dex.get("chains", []): - url = dex["url"].format( - token=token, - chain=chain, - chain_id=CHAIN_IDS.get(chain.lower(), 1), - ) - buttons.append(InlineKeyboardButton(f"{dex['emoji']} {dex['name']}", url=url)) - return buttons - def social_proof_text() -> str: """Dynamic social proof from DB stats.""" @@ -670,7 +663,7 @@ def scan_result_kb(address: str, chain: str) -> InlineKeyboardMarkup: InlineKeyboardButton("⭐ Watch", callback_data=f"watch_{address}_{chain}"), ], ] - dex_btns = dex_buttons(address, chain) + dex_btns = dex_buttons(address, chain, CHAIN_IDS.get(chain.lower(), 1)) for i in range(0, len(dex_btns), 2): rows.append(dex_btns[i : i + 2]) rows.append([InlineKeyboardButton("◀️ Menu", callback_data="menu_main")]) @@ -747,19 +740,16 @@ async def cmd_start(update: Update, ctx: ContextTypes.DEFAULT_TYPE): # Referral handling if ctx.args and ctx.args[0].startswith("ref_"): ref_code = ctx.args[0][4:] - if ref_code != u.get("referral_code") and not u.get("referred_by"): - conn = db.get_db() - referrer = conn.execute("SELECT user_id FROM users WHERE referral_code = ?", (ref_code,)).fetchone() - if referrer: - conn.execute( - "UPDATE users SET referred_by = ?, bonus_scans = bonus_scans + 5 WHERE user_id = ?", - (ref_code, user.id), - ) - conn.execute( - "UPDATE users SET bonus_scans = bonus_scans + 5 WHERE user_id = ?", - (referrer["user_id"],), - ) - conn.commit() + conn = db.get_db() + try: + apply_referral_code( + user.id, + ref_code, + u.get("referral_code"), + u.get("referred_by"), + conn, + ) + finally: conn.close() proof = social_proof_text() @@ -1346,24 +1336,19 @@ async def cmd_refer(update: Update, ctx: ContextTypes.DEFAULT_TYPE): user = update.effective_user u = db.get_or_create_user(user.id, user.username, user.first_name) ref = u.get("referral_code", "UNKNOWN") - link = f"https://t.me/{BOT_USERNAME}?start=ref_{ref}" + link = generate_referral_link(BOT_USERNAME, ref) conn = db.get_db() - count = conn.execute("SELECT COUNT(*) as c FROM users WHERE referred_by = ?", (ref,)).fetchone()["c"] - conn.close() - milestones = {5: "🥉", 10: "🥈", 25: "🥇", 50: "💎", 100: "👑"} - next_ms = next((k for k in sorted(milestones.keys()) if k > count), None) - ms_text = ( - f"\n🎯 Next milestone: {milestones[next_ms]} {next_ms} referrals" - if next_ms - else "\n👑 You've hit all milestones!" - ) + try: + stats = get_referral_stats(user.id, ref, conn) + finally: + conn.close() text = ( f"🤝 Refer & Earn\n{sep()}\n\n" f"Invite friends and both get 5 bonus scans!\n\n" f"🔗 Your Link:\n{link}\n\n" f"📊 Your Stats:\n" - f" Friends Referred: {count}\n" - f" Bonus Earned: {count * 5} scans{ms_text}\n\n" + f" Friends Referred: {stats['count']}\n" + f" Bonus Earned: {stats['bonus_earned']} scans{stats['next_milestone_text']}\n\n" f"How it works:\n" f" 1. Share your referral link\n" f" 2. Friend starts the bot via your link\n" @@ -1865,7 +1850,7 @@ async def handle_callback(update: Update, ctx: ContextTypes.DEFAULT_TYPE): await query.edit_message_text(text, parse_mode=ParseMode.HTML, reply_markup=back_kb()) elif data == "menu_refer": u = db.get_or_create_user(user.id) - link = f"https://t.me/{BOT_USERNAME}?start=ref_{u.get('referral_code', '')}" + link = generate_referral_link(BOT_USERNAME, u.get("referral_code", "")) await query.edit_message_text( f"🤝 Refer Friends\n\nShare your link:\n{link}\n\nBoth get 5 bonus scans!", parse_mode=ParseMode.HTML, diff --git a/app/domains/telegram/rugmunchbot/config.py b/app/domains/telegram/rugmunchbot/config.py index 443da5a..7360aeb 100644 --- a/app/domains/telegram/rugmunchbot/config.py +++ b/app/domains/telegram/rugmunchbot/config.py @@ -7,6 +7,8 @@ Tiers, channels, API endpoints, DEX ref links, and constants. import os +from app.domains.referral.partners import DEX_REF_LINKS # noqa: F401 + # ══════════════════════════════════════════════ # BOT # ══════════════════════════════════════════════ @@ -263,58 +265,7 @@ SCAN_PACKS = { # DEX REFERRAL LINKS # ══════════════════════════════════════════════ # These generate revenue when users trade through our links. -# Format: {platform: {url_template, chains, emoji, description}} -DEX_REF_LINKS = { - "jupiter": { - "name": "Jupiter", - "emoji": "🪐", - "url": "https://jup.ag/swap/SOL-{token}?ref=rugmunch", - "chains": ["solana"], - "description": "Best Solana DEX aggregator", - }, - "photon": { - "name": "Photon", - "emoji": "⚡", - "url": "https://photon-sol.tinyastro.io/en/lp/{token}?handle=rugmunch", - "chains": ["solana", "ethereum", "base"], - "description": "Fast Solana trading terminal", - }, - "bullx": { - "name": "BullX", - "emoji": "🐂", - "url": "https://neo.bullx.io/terminal?chain={chain}&address={token}&ref=rugmunch", - "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], - "description": "Multi-chain trading bot", - }, - "gmgn": { - "name": "GMGN.ai", - "emoji": "📊", - "url": "https://gmgn.ai/sol/token/{token}?ref=rugmunch", - "chains": ["solana", "ethereum", "base"], - "description": "Smart money tracking + trading", - }, - "birdeye": { - "name": "Birdeye", - "emoji": "🦅", - "url": "https://birdeye.so/token/{token}?ref=rugmunch", - "chains": ["solana", "ethereum", "base", "bsc", "arbitrum"], - "description": "Token analytics & charts", - }, - "oneinch": { - "name": "1inch", - "emoji": "🦄", - "url": "https://app.1inch.io/#/{chain_id}/simple/swap/ETH/{token}?ref=rugmunch", - "chains": ["ethereum", "base", "arbitrum", "bsc", "polygon", "avalanche"], - "description": "Best EVM DEX aggregator", - }, - "pancakeswap": { - "name": "PancakeSwap", - "emoji": "🥞", - "url": "https://pancakeswap.finance/swap?inputCurrency=BNB&outputCurrency={token}&ref=rugmunch", - "chains": ["bsc", "ethereum", "arbitrum", "base"], - "description": "BSC's #1 DEX", - }, -} +# Canonical definitions live in app.domains.referral.partners. # Chain ID mapping for 1inch CHAIN_IDS = { diff --git a/app/intel_pipeline.py b/app/intel_pipeline.py index ff2c30f..58b30bd 100644 --- a/app/intel_pipeline.py +++ b/app/intel_pipeline.py @@ -1,298 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.intel_pipeline. + +Migrate imports from `app.intel_pipeline` to `app.domains.intelligence.intel_pipeline`. +This shim will be removed in Phase 3 cleanup. """ -RMI Intelligence Pipeline - HF + Supabase + RAG -================================================ -Unified intelligence service: scam classification (HF models), -wallet labeling via RAG pattern matching, Supabase hybrid storage. -""" +from __future__ import annotations -import hashlib -import logging -import os -from datetime import UTC, datetime - -import httpx -from dotenv import load_dotenv - -load_dotenv("/app/.env", override=True) - -logger = logging.getLogger(__name__) - -HF_TOKEN = os.getenv("HF_TOKEN", "") -HF_API = "https://api-inference.huggingface.co/models" - - -def _get_url(): - return os.getenv("SUPABASE_URL", "") - - -def _get_key(): - return os.getenv("SUPABASE_SERVICE_KEY", "") - - -def _get_headers(): - key = _get_key() - return { - "apikey": key, - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - -# ── HF Models (Paywalled - using local fallback) ────── -# HF Inference API now requires PRO subscription ($9/mo). -# Using local pattern matching + RAG memory instead. -# Enable HF by setting HUGGINGFACE_TOKEN to a valid PRO key. - -SCAM_LABELS = ["scam", "rugpull", "honeypot", "legitimate", "phishing", "ponzi"] - -SCAM_KEYWORDS = { - "scam": ["scam", "fraud", "stole", "stolen", "exit scam", "fake"], - "rugpull": ["rug", "rugpull", "pulled liquidity", "drained", "removed liquidity", "lp removed"], - "honeypot": [ - "honeypot", - "cannot sell", - "can't sell", - "unable to sell", - "transfer disabled", - "sell tax 100", - ], - "phishing": [ - "phishing", - "airdrop scam", - "claim reward", - "verify wallet", - "seed phrase", - "private key", - ], - "ponzi": [ - "ponzi", - "mlm", - "multi level", - "referral rewards", - "guaranteed returns", - "double your", - ], - "insider": ["insider", "team wallet", "dev wallet", "pre-sale", "unlocked tokens", "vesting"], -} - - -async def classify_scam_risk(text: str) -> dict: - """Classify scam risk using local pattern matching (HF paywalled).""" - text_lower = text.lower() - scores = {} - - for label, keywords in SCAM_KEYWORDS.items(): - score = sum(1 for kw in keywords if kw in text_lower) - if score > 0: - scores[label] = min(score / len(keywords), 1.0) - - try: - from app.rag_service import search_documents - - rag_results = await search_documents("known_scams", text, limit=3) - for r in rag_results: - content = r.get("content", "").lower() - for label, keywords in SCAM_KEYWORDS.items(): - if any(kw in content for kw in keywords): - scores[label] = scores.get(label, 0) + 0.1 - except Exception: - pass - - if not scores: - return { - "risk": "low", - "labels": {}, - "confidence": 0, - "is_scam": False, - "risk_level": "low", - "model": "local_pattern_match", - } - - top = max(scores, key=scores.get) - top_score = scores[top] - is_scam = top in ("scam", "rugpull", "honeypot", "phishing", "ponzi") - - return { - "model": "local_pattern_match", - "labels": {k: round(v, 3) for k, v in sorted(scores.items(), key=lambda x: x[1], reverse=True)}, - "top_label": top, - "confidence": round(top_score, 3), - "is_scam": is_scam, - "risk_level": "high" if (is_scam and top_score > 0.5) else "medium" if is_scam else "low", - } - - -async def generate_embedding(text: str) -> list[float] | None: - """Generate embedding vector for RAG storage using HF free tier.""" - if not HF_TOKEN: - return None - - async with httpx.AsyncClient(timeout=60) as client: - r = await client.post( - f"{HF_API}/{EMBEDDING_MODEL}", # noqa: F821 -- pre-existing bug, see fix(f821) tracking issue - headers={"Authorization": f"Bearer {HF_TOKEN}"}, - json={"inputs": text[:1024]}, - ) - if r.status_code == 503: - logger.warning("HF embedding cold start, model loading") - return None - if r.status_code == 200: - result = r.json() - # Handle different response formats - if isinstance(result, list) and len(result) > 0: - return result[0] if isinstance(result[0], list) else result - return result - logger.warning(f"HF embedding failed: {r.status_code}") - return None - - -# ── Wallet Labeling via Pattern Memory ────────────────── - -WALLET_PATTERNS = { - "sybil_farmer": [ - "funded from exchange within seconds of 100+ other wallets", - "identical funding amounts across multiple wallets", - "no organic activity, only test transactions", - "funded by known Sybil distributor", - ], - "wash_trader": [ - "circular transfers between related wallets", - "buys and sells same token within minutes", - "volume spikes without holder count changes", - "coordinated buy/sell patterns", - ], - "sandwich_bot": [ - "high frequency trading with frontrun pattern", - "buys before large buys, sells immediately after", - "MEV extraction patterns", - "flashbots bundle usage", - ], - "liquidity_remover": [ - "large LP removal shortly after token launch", - "multiple LP positions removed simultaneously", - "liquidity drained to fresh wallet", - "LP removal preceded by marketing push", - ], - "honeypot_deployer": [ - "deploys tokens that can't be sold", - "reuses contract code across multiple tokens", - "disables transfers after liquidity added", - "ownership not renounced, hidden mint functions", - ], - "phishing_operator": [ - "sends tokens to many addresses with scam links", - "impersonates legitimate token contracts", - "uses airdrop as phishing vector", - "connects to known phishing domains", - ], - "mixer_user": [ - "funds pass through Tornado Cash or similar", - "receives from mixer, sends to clean wallet", - "layered mixing through multiple hops", - "funds originate from high-risk sources", - ], - "insider_trader": [ - "buys tokens before public announcements", - "linked to team wallets or deployers", - "sells immediately after hype peak", - "coordinated timing with other insiders", - ], -} - - -async def label_wallet(wallet_data: dict) -> dict: - """Label a wallet based on behavioral patterns and RAG memory.""" - labels = [] - confidence_scores = {} - - # Check behavioral patterns - behavior = wallet_data.get("behavior_summary", "") - wallet_data.get("transactions", []) - - for label, patterns in WALLET_PATTERNS.items(): - score = 0 - for pattern in patterns: - if pattern.lower() in behavior.lower(): - score += 1 - if score > 0: - confidence = min(score / len(patterns), 1.0) - confidence_scores[label] = round(confidence, 2) - if confidence > 0.3: - labels.append({"label": label, "confidence": round(confidence, 2)}) - - # Query RAG for similar wallet patterns - try: - from app.rag_service import search_documents - - rag_results = await search_documents( - "wallet_profiles", wallet_data.get("address", "") + " " + behavior, limit=5 - ) - if rag_results: - for r in rag_results: - content = r.get("content", "") - for label, patterns in WALLET_PATTERNS.items(): - if any(p.lower() in content.lower() for p in patterns): - if label not in confidence_scores: - confidence_scores[label] = 0 - confidence_scores[label] = min(confidence_scores[label] + 0.15, 1.0) - except Exception: - pass - - labels = [ - {"label": k, "confidence": v} - for k, v in sorted(confidence_scores.items(), key=lambda x: x[1], reverse=True) - if v > 0.2 - ] - - return { - "wallet_address": wallet_data.get("address", "unknown"), - "labels": labels[:5], - "primary_label": labels[0]["label"] if labels else "unknown", - "risk_score": max([line["confidence"] for line in labels]) * 100 if labels else 0, - "analyzed_at": datetime.now(UTC).isoformat(), - } - - -# ── Supabase Hybrid Storage ──────────────────────────── - - -async def sync_to_supabase(collection: str, document: dict) -> dict: - """Sync RAG document to Supabase for persistent hybrid storage.""" - if not _get_url() or not _get_key(): - return {"status": "skipped", "reason": "No Supabase config"} - - doc_id = hashlib.sha256(f"{collection}:{document.get('content', '')[:100]}".encode()).hexdigest()[:16] - - payload = { - "document_id": doc_id, - "collection": collection, - "content": document.get("content", "")[:5000], - "metadata": document.get("metadata", {}), - "synced_at": datetime.now(UTC).isoformat(), - } - - headers = _get_headers() - headers["Prefer"] = "resolution=merge-duplicates" - - async with httpx.AsyncClient(timeout=15) as client: - r = await client.post(f"{_get_url()}/rest/v1/rag_documents", json=payload, headers=headers) - if r.status_code in (200, 201, 409): - return {"status": "synced", "doc_id": doc_id, "supabase_status": r.status_code} - return {"status": "failed", "error": r.text[:200]} - - -# ── Batch Processing ─────────────────────────────────── - - -async def run_intelligence_cycle() -> dict: - """Run a full intelligence cycle: classify, label, embed, sync.""" - results = { - "cycle": datetime.now(UTC).isoformat(), - "scam_checks": 0, - "wallet_labels": 0, - "embeddings": 0, - "supabase_syncs": 0, - "errors": [], - } - - return results +from app.domains.intelligence.intel_pipeline import * # noqa: F403 diff --git a/app/mcp/__init__.py b/app/mcp/__init__.py index ac13c71..593e3b3 100644 --- a/app/mcp/__init__.py +++ b/app/mcp/__init__.py @@ -1,10 +1,36 @@ -# MCP Server Modules -# ================== -# x402 gateways run on Cloudflare Workers (7 chains, 45 tools each) -# The port 8001 local server was shut down - CF Workers handle everything -# Backend is at /srv/rugmuncher-backend/backend/ (Docker container rmi_backend) +"""Deprecated shim - re-exports app.domains.mcp public API. -# Legacy reference only - do not import for production use -# from .x402_mcp_server import X402MCPServer, X402ToolManager +Migrate imports from `app.mcp.*` to `app.domains.mcp.*`. +This shim will be removed in Phase 3 cleanup. +""" +from __future__ import annotations -__all__ = [] +from app.domains.mcp import ( + MCP_PROTOCOL_VERSION, + MCP_SERVER_VERSION, + TOOL_CATALOG, + TOOL_CATEGORIES, + TOOL_DEPRECATED, + TOOL_SUCCESSORS, + TOOL_VERSIONS, + TOOLS, + X402ToolManager, + call_tool, + get_tool_by_name, + handle_mcp_call, +) + +__all__ = [ + "MCP_PROTOCOL_VERSION", + "MCP_SERVER_VERSION", + "TOOLS", + "TOOL_CATALOG", + "TOOL_CATEGORIES", + "TOOL_DEPRECATED", + "TOOL_SUCCESSORS", + "TOOL_VERSIONS", + "X402ToolManager", + "call_tool", + "get_tool_by_name", + "handle_mcp_call", +] diff --git a/app/meme_intelligence.py b/app/meme_intelligence.py index 7201df4..e83487c 100644 --- a/app/meme_intelligence.py +++ b/app/meme_intelligence.py @@ -1,423 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.meme_intelligence. + +Migrate imports from `app.meme_intelligence` to `app.domains.intelligence.meme_intelligence`. +This shim will be removed in Phase 3 cleanup. """ -Meme Intelligence Platform - Meme Coin Tracking, Smart Money in Memes, -Big Wins/Losses, KOL Scorecards, Social Monitoring. +from __future__ import annotations -Integrations: -- DexScreener: meme token launches, trending -- LunarCrush: social sentiment, social dominance -- X/Twitter: KOL posts, viral content -- Telegram: channel monitoring, group sentiment -- Arkham: whale tracking in memes -- Birdeye: Solana meme tokens -- Pump.fun: new meme launches -""" - -import logging -import os - -from dotenv import load_dotenv - -load_dotenv("/app/.env", override=True) -from datetime import UTC, datetime, timedelta # noqa: E402 - -logger = logging.getLogger(__name__) - -# ── Meme Intelligence Data Structures ───────────────────────── - -MEME_CHAINS = ["solana", "ethereum", "base", "bsc", "arbitrum"] - -MEME_CATEGORIES = { - "dog": ["dog", "doge", "shib", "akita", "kishu"], - "cat": ["cat", "pepe", "mog", "meow"], - "politi": ["trump", "biden", "maga", "politics"], - "celeb": ["celeb", "influencer", "famous"], - "ai": ["ai", "gpt", "neural", "chat"], - "gaming": ["game", "gaming", "nft", "metaverse"], - "other": [], -} - -# KOL Database Schema -KOL_DATABASE = { - # Example structure - populate from research - "twitter_handles": [], - "telegram_channels": [], - "wallet_addresses": [], - "track_record": {}, # past calls, win rate - "follower_counts": {}, - "engagement_rates": {}, -} - -# ── Meme Token Intelligence ──────────────────────────────────── - - -async def get_meme_trending(limit: int = 50) -> list[dict]: - """Get trending meme tokens across chains.""" - from app.unified_provider import get_unified_provider - - provider = get_unified_provider() - memes = [] - - for chain in MEME_CHAINS: - try: - # Get trending from DexScreener - trending = await provider.get_dexscreener_trending(chain) - - for token in trending[:20]: - # Classify as meme based on name/symbol - category = classify_meme_category( - token.get("baseToken", {}).get("symbol", ""), - token.get("baseToken", {}).get("name", ""), - ) - - if category != "other": - memes.append( - { - "address": token.get("baseToken", {}).get("address"), - "symbol": token.get("baseToken", {}).get("symbol"), - "name": token.get("baseToken", {}).get("name"), - "chain": chain, - "category": category, - "price_usd": token.get("priceUsd"), - "volume_24h": token.get("volume", {}).get("h24"), - "price_change_24h": token.get("priceChange", {}).get("h24"), - "liquidity_usd": token.get("liquidity", {}).get("usd"), - "fdv": token.get("fdv"), - "pair_age_hours": token.get("pairCreatedAt"), - "is_meme": True, - } - ) - except Exception as e: - logger.debug(f"Error getting meme trending for {chain}: {e}") - - # Sort by volume - memes.sort(key=lambda x: x.get("volume_24h", 0), reverse=True) - - return memes[:limit] - - -async def get_smart_money_in_memes(limit: int = 20) -> list[dict]: - """Track known smart money wallets trading memes.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Query for smart money activities in meme tokens - result = ( - supabase.table("smart_money_activities") - .select(""" - *, - wallets ( - wallet_address, - wallet_label, - wallet_category, - win_rate, - total_pnl_usd - ) - """) - .eq("is_meme", True) - .order("amount_usd", desc=True) - .limit(limit) - .execute() - ) - - return result.data or [] - - -async def get_meme_wins_losses(period: str = "24h") -> dict: - """Get biggest wins and losses in meme tokens.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Calculate time range - if period == "24h": - time_range = datetime.now(UTC) - timedelta(hours=24) - elif period == "7d": - time_range = datetime.now(UTC) - timedelta(days=7) - elif period == "30d": - time_range = datetime.now(UTC) - timedelta(days=30) - else: - time_range = datetime.now(UTC) - timedelta(hours=24) - - # Get biggest wins - wins = ( - supabase.table("whale_movements") - .select("*") - .gte("collected_at", time_range.isoformat()) - .eq("transaction_type", "sell") - .order("amount_usd", desc=True) - .limit(20) - .execute() - ) - - # Get biggest losses - losses = ( - supabase.table("whale_movements") - .select("*") - .gte("collected_at", time_range.isoformat()) - .eq("transaction_type", "buy") - .order("amount_usd", desc=True) - .limit(20) - .execute() - ) - - return { - "period": period, - "wins": wins.data or [], - "losses": losses.data or [], - } - - -def classify_meme_category(symbol: str, name: str) -> str: - """Classify meme token into category.""" - text = f"{symbol} {name}".lower() - - for category, keywords in MEME_CATEGORIES.items(): - if any(kw in text for kw in keywords): - return category - - return "other" - - -# ── KOL Intelligence ────────────────────────────────────────── - - -async def get_kol_scorecard(kol_handle: str) -> dict | None: - """Get KOL scorecard with track record.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Get KOL profile - result = supabase.table("kols").select("*").eq("twitter_handle", kol_handle).execute() - - if not result.data: - return None - - kol = result.data[0] - - # Get their past calls - calls = ( - supabase.table("kol_calls") - .select("*") - .eq("kol_id", kol["id"]) - .order("called_at", desc=True) - .limit(50) - .execute() - ) - - # Calculate stats - total_calls = len(calls.data) if calls.data else 0 - winning_calls = len([c for c in (calls.data or []) if c.get("pnl_pct", 0) > 0]) - win_rate = (winning_calls / total_calls * 100) if total_calls > 0 else 0 - - avg_pnl = sum([c.get("pnl_pct", 0) for c in (calls.data or [])]) / total_calls if total_calls > 0 else 0 - - return { - "kol": kol, - "stats": { - "total_calls": total_calls, - "winning_calls": winning_calls, - "win_rate": round(win_rate, 2), - "average_pnl": round(avg_pnl, 2), - "follower_count": kol.get("follower_count"), - "engagement_rate": kol.get("engagement_rate"), - }, - "recent_calls": calls.data[:10] if calls.data else [], - } - - -async def get_top_kols_by_category(category: str = "memes", limit: int = 20) -> list[dict]: - """Get top KOLs by category with scorecards.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - result = ( - supabase.table("kols") - .select("*") - .eq("primary_category", category) - .order("win_rate", desc=True) - .order("follower_count", desc=True) - .limit(limit) - .execute() - ) - - return result.data or [] - - -# ── Social Monitoring ───────────────────────────────────────── - - -async def monitor_social_sentiment(token_address: str) -> dict: - """Monitor social sentiment for a token.""" - # LunarCrush integration - try: - from app.lunarcrush_connector import get_lunarcrush_connector - - lc = get_lunarcrush_connector() - - sentiment = await lc.get_sentiment(token_address) - - return { - "token": token_address, - "social_volume": sentiment.get("social_volume"), - "social_dominance": sentiment.get("social_dominance"), - "sentiment_score": sentiment.get("sentiment_score"), - "mentions_24h": sentiment.get("mentions_24h"), - "mentions_change_24h": sentiment.get("mentions_change_24h"), - } - except Exception: - return {"error": "LunarCrush not available"} - - -async def get_viral_crypto_posts(hours: int = 24) -> list[dict]: - """Get viral crypto posts from X/Twitter.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - time_range = datetime.now(UTC) - timedelta(hours=hours) - - result = ( - supabase.table("viral_posts") - .select("*") - .gte("posted_at", time_range.isoformat()) - .order("engagement_score", desc=True) - .limit(50) - .execute() - ) - - return result.data or [] - - -# ── Hack/Drain Alerts ──────────────────────────────────────── - - -async def get_recent_hacks_drains(hours: int = 24) -> list[dict]: - """Get recent hacks and drains from monitoring.""" - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - time_range = datetime.now(UTC) - timedelta(hours=hours) - - result = ( - supabase.table("security_alerts") - .select("*") - .gte("detected_at", time_range.isoformat()) - .in_("alert_type", ["hack", "drain", "exploit", "rugpull"]) - .order("amount_usd", desc=True) - .limit(50) - .execute() - ) - - return result.data or [] - - -async def format_hack_alert_for_social(hack_data: dict) -> dict: - """Format hack alert for X/Telegram posting.""" - return { - "x_post": f"""🚨 HACK ALERT 🚨 - -Protocol: {hack_data.get("protocol_name", "Unknown")} -Amount: ${hack_data.get("amount_usd", 0):,.0f} -Chain: {hack_data.get("chain", "Unknown")} -Type: {hack_data.get("attack_type", "Unknown")} - -{hack_data.get("description", "")[:200]} - -#CryptoSecurity #DeFi #HackAlert""", - "telegram_post": f"""🚨 *HACK ALERT* 🚨 - -*Protocol:* {hack_data.get("protocol_name", "Unknown")} -*Amount:* ${hack_data.get("amount_usd", 0):,.0f} -*Chain:* {hack_data.get("chain", "Unknown")} -*Type:* {hack_data.get("attack_type", "Unknown")} - -{hack_data.get("description", "")[:500]} - -Stay safe out there! 🔒""", - "severity": hack_data.get("severity", "medium"), - "amount_usd": hack_data.get("amount_usd", 0), - } - - -# ── Wallet Screenshot Generation ────────────────────────────── - - -async def generate_wallet_screenshot(wallet_address: str, pnl_data: dict) -> str: - """Generate wallet PnL screenshot for sharing.""" - # This would use a graphics API (Alibaba, etc.) to generate images - # For now, return placeholder - - return { - "wallet": wallet_address, - "total_pnl": pnl_data.get("total_pnl_usd", 0), - "win_rate": pnl_data.get("win_rate", 0), - "top_wins": pnl_data.get("top_wins", []), - "top_losses": pnl_data.get("top_losses", []), - "image_url": f"/api/v1/images/wallet/{wallet_address}/pnl-summary", - "share_url": f"https://rugmunch.io/wallet/{wallet_address}", - } - - -# ── Content Generation ──────────────────────────────────────── - - -async def generate_marketing_content(content_type: str, data: dict) -> dict: - """Generate marketing content using AI.""" - # This would call Alibaba's AI API for content generation - - templates = { - "win_announcement": """ -🎉 BIG WIN ALERT! 🎉 - -Wallet: {wallet_address[:8]}...{wallet_address[-6:]} -Token: {token_symbol} -Profit: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) - -This whale called it early and rode it all the way up! 🐋 - -Track smart money: https://rugmunch.io/wallet/{wallet_address} - -#Crypto #MemeCoin #SmartMoney -""", - "loss_announcement": """ -💀 LOSS PORN 💀 - -Wallet: {wallet_address[:8]}...{wallet_address[-6:]} -Token: {token_symbol} -Loss: ${pnl_usd:,.0f} ({pnl_pct:.1f}%) - -Oof. Another reminder to take profits! 📉 - -Learn from their mistakes: https://rugmunch.io/wallet/{wallet_address} - -#Crypto #Trading #LossPorn -""", - "kol_scorecard": """ -📊 KOL SCORECARD: @{kol_handle} - -Win Rate: {win_rate:.1f}% -Total Calls: {total_calls} -Avg PnL: {avg_pnl:.1f}% -Followers: {follower_count:,} - -Track their calls: https://rugmunch.io/kol/{kol_handle} - -#CryptoTwitter #KOL #Alpha -""", - } - - template = templates.get(content_type, "") - - # Format template with data - content = template.format(**data) if template else "" - - return { - "content_type": content_type, - "x_post": content[:280], - "telegram_post": content, - "generated_at": datetime.now(UTC).isoformat(), - } +from app.domains.intelligence.meme_intelligence import * # noqa: F403 diff --git a/app/mount.py b/app/mount.py index 11095e2..8f93d55 100644 --- a/app/mount.py +++ b/app/mount.py @@ -36,7 +36,7 @@ ROUTER_MODULES: Final[list[str]] = [ "app.api.v1.admin.alerts_webhook", # /api/v1/admin/alerts/webhook "app.api.v1.admin.glitchtip_test", # /api/v1/_test/* (T07) "app.api.v1.catalog", # /api/v1/catalog/* - "app.api.v1.mcp", # /mcp/* (JSON-RPC + plain JSON) + "app.domains.mcp.router", # /mcp/* (JSON-RPC + plain JSON) # Domain facades (per v4.0 T28-T34) "app.domains.news", # /api/v1/news/* "app.domains.news.admin_router", # /api/v1/news/_admin/* diff --git a/app/n8n_intelligence_workflows.py b/app/n8n_intelligence_workflows.py index 763ec33..359e5cb 100644 --- a/app/n8n_intelligence_workflows.py +++ b/app/n8n_intelligence_workflows.py @@ -1,438 +1,8 @@ +"""Deprecated shim - re-exports app.domains.intelligence.n8n_intelligence_workflows. + +Migrate imports from `app.n8n_intelligence_workflows` to `app.domains.intelligence.n8n_intelligence_workflows`. +This shim will be removed in Phase 3 cleanup. """ -n8n Workflow Automation - Market Intelligence & Premium Data Pulls. -Scheduled workflows for efficient data collection from multiple sources. -Runs every 30-60 minutes based on scan volume and data freshness needs. +from __future__ import annotations -Integrations: -- CoinGecko: trending, global metrics, top gainers/losers -- DexScreener: new pairs, trending tokens, volume spikes -- Dune Analytics: custom queries for whale tracking, scam patterns -- Helius: token mints, large transfers, new token deployments -- Moralis: EVM whale movements, new contract deployments -- Arkham: entity labeling, exchange flows -- Nansen: smart money tracking (if available) -- Forta: real-time threat detection -""" - -import logging -from datetime import UTC, datetime - -logger = logging.getLogger(__name__) - -# ── n8n Workflow Definitions ───────────────────────────────── - -N8N_BASE_URL = "https://n8n.rugmunch.io" -N8N_WEBHOOK_URL = f"{N8N_BASE_URL}/webhook" - -# Market Intelligence Workflows -MARKET_INTEL_WORKFLOWS = { - "trending_tokens": { - "name": "Trending Tokens Collector", - "schedule": "*/30 * * * *", # Every 30 minutes - "sources": ["coingecko", "dexscreener", "geckoterminal"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/trending", - "description": "Collect trending tokens from multiple DEXs", - "output_table": "market_trending_tokens", - }, - "whale_movements": { - "name": "Whale Movement Tracker", - "schedule": "*/15 * * * *", # Every 15 minutes (high priority) - "sources": ["helius", "moralis", "arkham"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/whales", - "description": "Track large transfers across chains", - "output_table": "whale_movements", - }, - "new_token_deployments": { - "name": "New Token Deployments", - "schedule": "*/10 * * * *", # Every 10 minutes (fast detection) - "sources": ["helius", "moralis", "dexscreener"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/new-tokens", - "description": "Detect new token deployments in real-time", - "output_table": "new_token_deployments", - }, - "volume_spikes": { - "name": "Volume Spike Detector", - "schedule": "*/5 * * * *", # Every 5 minutes (critical) - "sources": ["dexscreener", "geckoterminal", "birdeye"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/volume-spikes", - "description": "Detect unusual volume increases", - "output_table": "volume_spikes", - }, - "global_metrics": { - "name": "Market Global Metrics", - "schedule": "0 * * * *", # Every hour - "sources": ["coingecko"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/global", - "description": "Global crypto market metrics", - "output_table": "market_global_metrics", - }, - "defi_protocols": { - "name": "DeFi Protocol Analytics", - "schedule": "0 */2 * * *", # Every 2 hours - "sources": ["defillama", "dune"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/defi", - "description": "DeFi TVL, volume, user metrics", - "output_table": "defi_protocol_metrics", - }, - "nft_market": { - "name": "NFT Market Intelligence", - "schedule": "0 */4 * * *", # Every 4 hours - "sources": ["alchemy", "opensea_api"], - "endpoint": f"{N8N_WEBHOOK_URL}/market/nft", - "description": "NFT floor prices, volume, trends", - "output_table": "nft_market_metrics", - }, -} - -# Premium Intelligence Workflows -PREMIUM_INTEL_WORKFLOWS = { - "smart_money_tracking": { - "name": "Smart Money Tracker", - "schedule": "*/30 * * * *", # Every 30 minutes - "sources": ["nansen", "arkham", "dune"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/smart-money", - "description": "Track known smart money wallets", - "output_table": "smart_money_activities", - "tier": "premium", - }, - "exchange_flows": { - "name": "Exchange Flow Analysis", - "schedule": "*/15 * * * *", # Every 15 minutes - "sources": ["arkham", "dune", "glassnode"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/exchange-flows", - "description": "Track exchange inflows/outflows", - "output_table": "exchange_flows", - "tier": "premium", - }, - "insider_trading": { - "name": "Insider Trading Detection", - "schedule": "*/20 * * * *", # Every 20 minutes - "sources": ["dune", "arkham", "helius"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/insider", - "description": "Detect potential insider trading patterns", - "output_table": "insider_trading_alerts", - "tier": "premium_plus", - }, - "launchpad_monitor": { - "name": "Launchpad Monitor", - "schedule": "*/10 * * * *", # Every 10 minutes - "sources": ["dexscreener", "pumpfun_api", "geckoterminal"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/launchpad", - "description": "Monitor new launches across platforms", - "output_table": "launchpad_monitoring", - "tier": "premium", - }, - "cluster_analysis": { - "name": "Cluster Pattern Analysis", - "schedule": "0 */2 * * *", # Every 2 hours - "sources": ["internal_clustering", "dune"], - "endpoint": f"{N8N_WEBHOOK_URL}/premium/clusters", - "description": "Deep cluster pattern analysis", - "output_table": "cluster_patterns", - "tier": "premium_plus", - }, -} - -# Security Intelligence Workflows -SECURITY_INTEL_WORKFLOWS = { - "scam_detection": { - "name": "Real-time Scam Detection", - "schedule": "*/5 * * * *", # Every 5 minutes (critical) - "sources": ["forta", "internal_ml", "community_reports"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/scams", - "description": "Detect new scam patterns", - "output_table": "scam_alerts", - "priority": "critical", - }, - "rugpull_detection": { - "name": "Rugpull Detection", - "schedule": "*/2 * * * *", # Every 2 minutes (ultra-critical) - "sources": ["internal_ml", "liquidity_monitor"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/rugpulls", - "description": "Detect rugpull patterns in real-time", - "output_table": "rugpull_alerts", - "priority": "critical", - }, - "contract_vulnerabilities": { - "name": "Contract Vulnerability Scanner", - "schedule": "0 * * * *", # Every hour - "sources": ["slither", "mythril", "internal_scanner"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/vulnerabilities", - "description": "Scan new contracts for vulnerabilities", - "output_table": "contract_vulnerabilities", - "priority": "high", - }, - "phishing_detection": { - "name": "Phishing Domain Detection", - "schedule": "0 */6 * * *", # Every 6 hours - "sources": ["guardian", "cryptoscamdb", "community"], - "endpoint": f"{N8N_WEBHOOK_URL}/security/phishing", - "description": "Detect new phishing domains", - "output_table": "phishing_domains", - "priority": "high", - }, -} - - -# ── n8n Workflow Execution ──────────────────────────────────── - - -async def trigger_n8n_workflow(workflow_name: str, data: dict | None = None) -> bool: - """Trigger an n8n workflow via webhook.""" - import httpx - - # Find workflow config - all_workflows = { - **MARKET_INTEL_WORKFLOWS, - **PREMIUM_INTEL_WORKFLOWS, - **SECURITY_INTEL_WORKFLOWS, - } - workflow = all_workflows.get(workflow_name) - - if not workflow: - logger.error(f"Workflow not found: {workflow_name}") - return False - - webhook_url = workflow["endpoint"] - - try: - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.post( - webhook_url, - json={ - "workflow": workflow_name, - "triggered_at": datetime.now(UTC).isoformat(), - "data": data or {}, - }, - ) - - if response.status_code in (200, 201, 202): - logger.info(f"Triggered workflow: {workflow_name}") - return True - else: - logger.error(f"Workflow trigger failed: {workflow_name} - {response.status_code}") - return False - - except Exception as e: - logger.error(f"Workflow trigger error: {workflow_name} - {e}") - return False - - -async def execute_market_intelligence_pull(workflow_name: str): - """Execute a market intelligence data pull.""" - workflow = MARKET_INTEL_WORKFLOWS.get(workflow_name) - if not workflow: - return - - logger.info(f"Executing market intel pull: {workflow['name']}") - - # Collect data from sources - collected_data = { - "workflow": workflow_name, - "sources": workflow["sources"], - "collected_at": datetime.now(UTC).isoformat(), - "data": {}, - } - - # Source-specific collection logic - for source in workflow["sources"]: - try: - if source == "coingecko": - from app.coingecko_connector import get_coingecko_connector - - connector = get_coingecko_connector() - - if "trending" in workflow_name.lower(): - trending = await connector.get_trending() - collected_data["data"]["coingecko_trending"] = trending - elif "global" in workflow_name.lower(): - global_data = await connector.get_global_metrics() - collected_data["data"]["coingecko_global"] = global_data - - elif source == "dexscreener": - from app.unified_provider import get_unified_provider - - provider = get_unified_provider() - - if "trending" in workflow_name.lower(): - trending = await provider.get_dexscreener_trending() - collected_data["data"]["dexscreener_trending"] = trending - elif "volume" in workflow_name.lower(): - spikes = await provider.get_volume_spikes() - collected_data["data"]["dexscreener_spikes"] = spikes - - elif source == "helius": - from app.chain_client import get_chain_client - - client = get_chain_client() - - if "new" in workflow_name.lower(): - new_tokens = await client.get_new_token_mints(limit=50) - collected_data["data"]["helius_new_tokens"] = new_tokens - elif "whale" in workflow_name.lower(): - whales = await client.get_large_transfers(limit=20) - collected_data["data"]["helius_whales"] = whales - - except Exception as e: - logger.error(f"Error collecting from {source}: {e}") - collected_data["data"][source] = {"error": str(e)} - - # Store in Supabase - await _store_intelligence_data(workflow["output_table"], collected_data) - - # Trigger alerts if needed - await _process_intelligence_alerts(workflow_name, collected_data) - - -async def _store_intelligence_data(table_name: str, data: dict): - """Store intelligence data in Supabase.""" - try: - import os - - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - # Insert data - supabase.table(table_name).insert( - { - "data": data, - "collected_at": data.get("collected_at"), - "workflow": data.get("workflow"), - } - ).execute() - - logger.info(f"Stored intelligence data in {table_name}") - - except Exception as e: - logger.error(f"Failed to store intelligence data: {e}") - - -async def _process_intelligence_alerts(workflow_name: str, data: dict): - """Process intelligence data and trigger alerts.""" - # Check for significant findings - if "whale" in workflow_name.lower(): - # Check for unusually large transfers - whales = data.get("data", {}).get("helius_whales", []) - for whale in whales: - amount = whale.get("amount", 0) - if amount > 1000: # >1000 SOL - await _create_alert("whale_movement", whale) - - elif "scam" in workflow_name.lower() or "rugpull" in workflow_name.lower(): - # Immediate alert for security issues - await _create_alert("security_critical", data) - - elif "volume" in workflow_name.lower(): - # Check for unusual volume spikes - spikes = data.get("data", {}).get("dexscreener_spikes", []) - for spike in spikes: - if spike.get("volume_change_pct", 0) > 500: # >500% increase - await _create_alert("volume_spike", spike) - - -async def _create_alert(alert_type: str, data: dict): - """Create an intelligence alert.""" - try: - import os - - from supabase import create_client - - supabase = create_client(os.getenv("SUPABASE_URL"), os.getenv("SUPABASE_KEY")) - - alert_data = { - "alert_type": alert_type, - "severity": "critical" if "security" in alert_type else "high", - "data": data, - "created_at": datetime.now(UTC).isoformat(), - "is_read": False, - } - - supabase.table("intelligence_alerts").insert(alert_data).execute() - - logger.info(f"Created intelligence alert: {alert_type}") - - except Exception as e: - logger.error(f"Failed to create alert: {e}") - - -# ── Dune Analytics Integration ──────────────────────────────── - - -async def execute_dune_query(query_id: str, params: dict | None = None) -> dict | None: - """Execute a Dune Analytics query.""" - import os - - import httpx - - dune_api_key = os.getenv("DUNE_API_KEY", "") - if not dune_api_key: - logger.warning("DUNE_API_KEY not configured") - return None - - try: - async with httpx.AsyncClient(timeout=60.0) as client: - # Execute query - response = await client.post( - f"https://api.dune.com/api/v1/query/{query_id}/execute", - headers={"X-Dune-API-Key": dune_api_key}, - json=params or {}, - ) - - if response.status_code != 200: - logger.error(f"Dune query failed: {response.status_code}") - return None - - execution_id = response.json().get("execution_id") - - # Wait for results - import asyncio - - for _ in range(10): # Max 10 attempts - await asyncio.sleep(2) - - result_response = await client.get( - f"https://api.dune.com/api/v1/execution/{execution_id}/results", - headers={"X-Dune-API-Key": dune_api_key}, - ) - - if result_response.status_code == 200: - result = result_response.json() - if result.get("state") == "QUERY_STATE_COMPLETED": - return result.get("result", {}).get("rows", []) - - return None - - except Exception as e: - logger.error(f"Dune query error: {e}") - return None - - -# Pre-configured Dune queries for RMI -DUNE_QUERIES = { - "ethereum_whale_transfers": { - "query_id": "1234567", # Replace with actual query ID - "description": "Track large ETH transfers from known whale wallets", - "schedule": "*/15 * * * *", - }, - "defi_protocol_volumes": { - "query_id": "2345678", - "description": "Daily DEX volumes across major protocols", - "schedule": "0 * * * *", - }, - "nft_wash_trading": { - "query_id": "3456789", - "description": "Detect potential NFT wash trading patterns", - "schedule": "0 */6 * * *", - }, - "stablecoin_flows": { - "query_id": "4567890", - "description": "Track USDC/USDT flows to/from exchanges", - "schedule": "*/30 * * * *", - }, - "new_contract_deployments": { - "query_id": "5678901", - "description": "New contract deployments with large funding", - "schedule": "*/10 * * * *", - }, -} +from app.domains.intelligence.n8n_intelligence_workflows import * # noqa: F403 diff --git a/app/news_intelligence.py b/app/news_intelligence.py index 2ef6565..de043ff 100644 --- a/app/news_intelligence.py +++ b/app/news_intelligence.py @@ -1,165 +1,8 @@ -#!/usr/bin/env python3 -""" -RMI News Intelligence v3 - Industry Best -========================================= -AI-powered news pipeline: categorization, sentiment, trending, briefing. -Uses MiniMax ($20/mo flat) + Ollama Cloud. +"""Deprecated shim - re-exports app.domains.intelligence.news_intelligence. + +Migrate imports from `app.news_intelligence` to `app.domains.intelligence.news_intelligence`. +This shim will be removed in Phase 3 cleanup. """ +from __future__ import annotations -import json -import logging -import os -import urllib.request -from collections import Counter -from datetime import UTC, datetime - -logger = logging.getLogger("rmi.news_v3") -OLLAMA_KEY = os.getenv("OLLAMA_API_KEY", os.getenv("DEEPSEEK_API_KEY", "")) -OLLAMA_URL = "https://ollama.com/v1/chat/completions" - -# Simple in-memory trending tracker -_trending_topics = Counter() -_breaking_alerts = [] - - -def analyze_article(title: str, content: str = "") -> dict: - """Full AI analysis of a news article.""" - text = f"{title} {content[:300]}" - - # Category (fast, cached) - from app.ai_pipeline_v3 import classify_news - - category = classify_news(title, content) - - # Sentiment via MiniMax (batched - 1 call per article is fine at flat rate) - sentiment = "neutral" - try: - k = os.getenv("OLLAMA_API_KEY", "") - if not k: - with open("/app/.env") as f: - for line in f: - if line.startswith("OLLAMA_API_KEY"): - k = line.strip().split("=", 1)[1] - break - if not k: - return {"category": category, "sentiment": "neutral", "is_breaking": False} - body = json.dumps( - { - "model": "deepseek-v4-flash", - "messages": [ - { - "role": "system", - "content": "Classify sentiment: BULLISH BEARISH NEUTRAL. Reply one word only.", - }, - {"role": "user", "content": text[:400]}, - ], - "max_tokens": 10, - "temperature": 0.1, - } - ).encode() - req = urllib.request.Request( - OLLAMA_URL, - data=body, - headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, - ) - resp = urllib.request.urlopen(req, timeout=8) - sentiment = json.loads(resp.read())["choices"][0]["message"]["content"].strip().upper() - if "BULL" in sentiment: - sentiment = "bullish" - elif "BEAR" in sentiment: - sentiment = "bearish" - else: - sentiment = "neutral" - except Exception as e: - logger.warning(f"Sentiment failed: {e}") - - # Track trending topics - for word in title.lower().split(): - if len(word) > 4 and word not in ( - "after", - "before", - "while", - "could", - "would", - "should", - "their", - "there", - "these", - "those", - "about", - "which", - ): - _trending_topics[word] += 1 - - # Detect breaking news - is_breaking = category == "SCAM" or any( - w in text.lower() for w in ["hacked", "exploited", "drained", "rug pulled", "emergency"] - ) - - return { - "category": category, - "sentiment": sentiment, - "is_breaking": is_breaking, - "analyzed_at": datetime.now(UTC).isoformat(), - } - - -def get_trending(limit: int = 10) -> list: - """Get trending topics from recent article analysis.""" - return [{"topic": word, "count": count} for word, count in _trending_topics.most_common(limit)] - - -def get_breaking() -> list: - """Get breaking news alerts.""" - return _breaking_alerts[-10:] - - -def daily_briefing() -> str: - """Generate an AI-powered daily news briefing using Ollama Cloud.""" - trending = get_trending(5) - topics = ", ".join(f"{t['topic']}({t['count']})" for t in trending) - - k = OLLAMA_KEY - body = json.dumps( - { - "model": "deepseek-v4-flash", - "messages": [ - { - "role": "system", - "content": "Write a 3-sentence daily crypto news briefing. Mention trending topics. Professional tone. Under 100 words.", - }, - {"role": "user", "content": f"Trending topics: {topics}"}, - ], - "max_tokens": 150, - "temperature": 0.5, - } - ).encode() - - try: - req = urllib.request.Request( - OLLAMA_URL, - data=body, - headers={"Authorization": f"Bearer {k}", "Content-Type": "application/json"}, - ) - resp = urllib.request.urlopen(req, timeout=15) - return json.loads(resp.read())["choices"][0]["message"]["content"].strip() - except Exception: - return f"Daily briefing: {topics} are trending in crypto news today." - - -def news_search(query: str, articles: list, limit: int = 10) -> list: - """Smart search across articles with relevance ranking.""" - results = [] - q_lower = query.lower() - for a in articles: - score = 0 - if q_lower in a.get("title", "").lower(): - score += 10 - if q_lower in a.get("content", "").lower(): - score += 5 - if q_lower in a.get("category", "").lower(): - score += 3 - if score > 0: - a["relevance"] = score - results.append(a) - return sorted(results, key=lambda x: x.get("relevance", 0), reverse=True)[:limit] +from app.domains.intelligence.news_intelligence import * # noqa: F403 diff --git a/app/prediction_market_service.py b/app/prediction_market_service.py index 0588baf..16ef225 100644 --- a/app/prediction_market_service.py +++ b/app/prediction_market_service.py @@ -1,1122 +1,13 @@ +"""Deprecated shim - re-exports app.domains.markets. + +Migrate imports from `app.prediction_market_service` to `app.domains.markets`. +This shim will be removed in Phase 3 cleanup. """ -RMI Prediction Market Intelligence Service -=========================================== -Multi-source prediction market data aggregation for crypto security intelligence. - -Data sources (all free, zero auth for read-only): - - Polymarket: Gamma API (search/discovery), CLOB API (prices/history), Data API (trades) - - Kalshi: REST API /series, /markets, /events, /orderbook (unauthenticated) - - Limitless: REST API /markets (Base chain, crypto-native) - - Manifold: REST API /markets (play-money, open-source sentiment signals) - -Open-source reference implementations (GitHub): - - homerun (braedonsaunders/homerun): Open-source prediction market platform for - Polymarket + Kalshi. Python strategies, backtesting, data sources, live trading. - - prediction-market-edge-bot: SX Bet + Polymarket aggregator with smart order routing. - - Awesome-Prediction-Market-Tools (aarora4): Curated directory of 50+ tools - including Oddpool (cross-venue aggregator), analytics dashboards, trading bots. - -Architecture: - - Direct external API calls (NEVER route through own API - anti-circular-dependency rule) - - All 4 sources queried in parallel with individual try/except - - Results normalized into unified PredictionMarket dataclass - - Redis caching: 30s TTL prices, 5min searches, 1hr digests - -Integration points: - - SENTINEL scanner: cross-reference token risk scores with market probability - - Wallet Memory Bank: entity/deployer reputation from prediction market odds - - RugMaps: visual correlation between market odds and on-chain wallet clusters - - x402 tools: expose as paid security intelligence endpoints - -Pitfalls: - - Polymarket Gamma API double-encodes outcomePrices/clobTokenIds as JSON strings - - One source timeout must not kill the entire call - individual try/except per source - - Prediction market data is probabilistic, not definitive - always cross-reference - with on-chain scanner results - - Don't poll every market every tick - use targeted search + category filters -""" - -import asyncio -import hashlib -import json -import logging -import os -from dataclasses import dataclass, field -from datetime import UTC, datetime - -import httpx - -logger = logging.getLogger("prediction_market") - -# ── API Endpoints ──────────────────────────────────────────────── - -POLYMARKET_GAMMA = "https://gamma-api.polymarket.com" -POLYMARKET_CLOB = "https://clob.polymarket.com" -POLYMARKET_DATA = "https://data-api.polymarket.com" - -KALSHI_BASE = "https://external-api.kalshi.com/trade-api/v2" - -LIMITLESS_BASE = "https://api.limitless.exchange" - -MANIFOLD_BASE = "https://api.manifold.markets/v0" - -# ── Category mappings for security relevance ──────────────────── - -SECURITY_KEYWORDS = [ - "hack", - "exploit", - "rug", - "scam", - "fraud", - "breach", - "leak", - "drain", - "phish", - "backdoor", - "vulnerability", - "zero-day", - "sanction", - "indict", - "arrest", - "freeze", - "seize", - "clampdown", - "depeg", - "insolvent", - "bankrupt", - "collapse", - "default", - "theft", - "heist", - "compromise", - "ransomware", - "malware", - "SEC", - "CFTC", - "DOJ", - "FBI", - "regulatory", - "enforcement", -] - -CRYPTO_KEYWORDS = [ - "bitcoin", - "ethereum", - "solana", - "crypto", - "defi", - "token", - "blockchain", - "web3", - "nft", - "stablecoin", - "usdt", - "usdc", - "dai", - "exchange", - "binance", - "coinbase", - "uniswap", - "aave", - "tether", - "circle", - "layer", - "L1", - "L2", - "rollup", - "bridge", - "polygon", - "arbitrum", - "optimism", - "avalanche", - "fantom", - "chainlink", - "makerdao", - "lido", - "eigenlayer", -] - -# ── Dataclasses ───────────────────────────────────────────────── - - -@dataclass -class PredictionMarket: - """Unified prediction market result across all sources.""" - - source: str # "polymarket" | "kalshi" | "limitless" | "manifold" - source_id: str # native ID from source (slug, ticker, etc.) - question: str - slug: str - probability_yes: float # 0.0-1.0 - probability_no: float # 0.0-1.0 - volume_usd: float - liquidity_usd: float = 0.0 - category: str = "" - tags: list[str] = field(default_factory=list) - tokens_mentioned: list[str] = field(default_factory=list) - is_security_relevant: bool = False - is_crypto_relevant: bool = False - url: str = "" - ends_at: str | None = None - updated_at: str = "" - - def __post_init__(self): - """Auto-classify relevance based on keywords in question.""" - q_lower = self.question.lower() - self.is_security_relevant = any(kw in q_lower for kw in SECURITY_KEYWORDS) - self.is_crypto_relevant = any(kw in q_lower for kw in CRYPTO_KEYWORDS) - - -@dataclass -class PredictionDigest: - """Daily intelligence digest of security-relevant prediction markets.""" - - generated_at: str - total_markets_searched: int - security_relevant_count: int - crypto_relevant_count: int - top_threats: list[PredictionMarket] = field(default_factory=list) - token_specific_markets: list[PredictionMarket] = field(default_factory=list) - ecosystem_risk_markets: list[PredictionMarket] = field(default_factory=list) - regulatory_markets: list[PredictionMarket] = field(default_factory=list) - - -# ── Service ───────────────────────────────────────────────────── - - -class PredictionMarketService: - """Multi-source prediction market data with parallel fetching + caching.""" - - def __init__(self, http_client: httpx.AsyncClient | None = None): - self._http = http_client or httpx.AsyncClient(timeout=15.0) - self._redis = None # Lazy init via get_redis() - - def _get_redis(self): - """Lazy Redis connection for caching. Returns None if unavailable.""" - if self._redis is not None: - return self._redis - try: - import redis.asyncio as aioredis - - self._redis = aioredis.from_url( - os.getenv("REDIS_URL", "redis://localhost:6379/0"), - decode_responses=True, - socket_connect_timeout=3, - ) - # Set to False if we couldn't actually connect - if self._redis is None: - self._redis = False - except Exception as e: - logger.warning(f"Redis unavailable, caching disabled: {e}") - self._redis = False - return self._redis if self._redis is not False else None - - # ── Public API ────────────────────────────────────────── - - async def search( - self, - query: str, - categories: list[str] | None = None, - min_volume: float = 0, - security_only: bool = False, - ) -> list[PredictionMarket]: - """Search all prediction market sources in parallel. - - Args: - query: Search term (token name, event, protocol, etc.) - categories: Optional filter by source categories - min_volume: Minimum USD volume to include - security_only: Only return security-relevant markets - """ - # Check Redis cache - cache_key = f"predmkt:search:{_cache_hash(query, categories, min_volume, security_only)}" - redis = self._get_redis() - if redis: - try: - cached = await redis.get(cache_key) - if cached: - markets_data = json.loads(cached) - return [_dict_to_market(d) for d in markets_data] - except Exception: - pass # Cache miss or Redis error - fall through to live query - - # Fire all 4 sources in parallel - results: list[list[PredictionMarket]] = await asyncio.gather( - self._search_polymarket(query), - self._search_kalshi(query), - self._search_limitless(query), - self._search_manifold(query), - return_exceptions=True, - ) - - # Flatten and handle exceptions - all_markets: list[PredictionMarket] = [] - sources = ["polymarket", "kalshi", "limitless", "manifold"] - for i, result in enumerate(results): - if isinstance(result, Exception): - logger.warning(f"{sources[i]} search failed: {result}") - continue - if isinstance(result, list): - all_markets.extend(result) - - # Filter - if min_volume > 0: - all_markets = [m for m in all_markets if m.volume_usd >= min_volume] - if security_only: - all_markets = [m for m in all_markets if m.is_security_relevant] - - # Sort by volume descending - all_markets.sort(key=lambda m: m.volume_usd, reverse=True) - - # Cache (5 min TTL) - if redis: - try: - await redis.setex(cache_key, 300, json.dumps([_market_to_dict(m) for m in all_markets[:50]])) - except Exception as e: - logger.warning(f"Redis cache write failed: {e}") - - return all_markets - - async def token_markets(self, symbol: str) -> list[PredictionMarket]: - """Find all prediction markets mentioning a specific token symbol.""" - results = await self.search(f'"{symbol}" token crypto', security_only=False) - - # Filter to markets actually about this token (mention in question) - symbol_lower = symbol.lower() - token_markets = [m for m in results if symbol_lower in m.question.lower()] - return token_markets - - async def security_digest(self) -> PredictionDigest: - """Generate daily intelligence digest of security-relevant prediction markets. - - Queries crypto + security keywords across all sources, categorizes - results by threat type: top threats, token-specific, ecosystem risk, - regulatory. - """ - cache_key = f"predmkt:digest:{datetime.now(UTC).strftime('%Y-%m-%d')}" - redis = self._get_redis() - if redis: - try: - cached = await redis.get(cache_key) - if cached: - data = json.loads(cached) - return _dict_to_digest(data) - except Exception: - pass # Cache miss or Redis error - fall through to live query - - # Search for security-relevant crypto markets - security_queries = [ - "crypto hack exploit scam", - "defi rug pull fraud", - "exchange insolvent breach", - "stablecoin depeg collapse", - "crypto regulation SEC enforcement", - "blockchain vulnerability zero-day", - ] - - all_markets: list[PredictionMarket] = [] - searches = [self.search(q, security_only=False) for q in security_queries] - results = await asyncio.gather(*searches, return_exceptions=True) - - for result in results: - if isinstance(result, list): - all_markets.extend(result) - - # De-duplicate by question similarity - seen_questions = set() - unique_markets = [] - for m in all_markets: - q_key = m.question.lower().strip()[:80] - if q_key not in seen_questions: - seen_questions.add(q_key) - unique_markets.append(m) - - # Categorize - top_threats = [m for m in unique_markets if m.is_security_relevant and m.volume_usd > 10000] - top_threats.sort(key=lambda m: m.volume_usd, reverse=True) - - token_specific = [ - m for m in unique_markets if m.is_crypto_relevant and m.is_security_relevant and len(m.tokens_mentioned) > 0 - ] - token_specific.sort(key=lambda m: m.volume_usd, reverse=True) - - ecosystem_risk = [ - m for m in unique_markets if m.is_crypto_relevant and not m.is_security_relevant and m.volume_usd > 50000 - ] - ecosystem_risk.sort(key=lambda m: m.volume_usd, reverse=True) - - regulatory = [ - m - for m in unique_markets - if m.is_security_relevant - and any(kw in m.question.lower() for kw in ["sec", "cftc", "doj", "regulation", "sanction", "ban"]) - ] - regulatory.sort(key=lambda m: m.volume_usd, reverse=True) - - digest = PredictionDigest( - generated_at=datetime.now(UTC).isoformat(), - total_markets_searched=len(unique_markets), - security_relevant_count=len([m for m in unique_markets if m.is_security_relevant]), - crypto_relevant_count=len([m for m in unique_markets if m.is_crypto_relevant]), - top_threats=top_threats[:20], - token_specific_markets=token_specific[:20], - ecosystem_risk_markets=ecosystem_risk[:10], - regulatory_markets=regulatory[:10], - ) - - # Cache (1 hour) - if redis: - try: - await redis.setex(cache_key, 3600, json.dumps(_digest_to_dict(digest))) - except Exception as e: - logger.warning(f"Redis digest cache write failed: {e}") - - return digest - - async def trending(self, limit: int = 20, source: str | None = None) -> list[PredictionMarket]: - """Get top trending prediction markets by volume across all sources.""" - # Fetch top events from each source in parallel - tasks = [] - if not source or source == "polymarket": - tasks.append(self._trending_polymarket(limit)) - else: - tasks.append(asyncio.sleep(0)) # placeholder - - if not source or source == "kalshi": - tasks.append(self._trending_kalshi(limit)) - else: - tasks.append(asyncio.sleep(0)) - - if not source or source == "limitless": - tasks.append(self._trending_limitless(limit)) - else: - tasks.append(asyncio.sleep(0)) - - results = await asyncio.gather(*tasks, return_exceptions=True) - - all_markets = [] - for result in results: - if isinstance(result, list): - all_markets.extend(result) - elif isinstance(result, Exception): - pass # Individual source failures logged in _trending_* methods - - all_markets.sort(key=lambda m: m.volume_usd, reverse=True) - return all_markets[:limit] - - async def market_detail(self, source: str, market_id: str) -> PredictionMarket | None: - """Get detailed data for a specific market including orderbook.""" - if source == "polymarket": - return await self._polymarket_detail(market_id) - elif source == "kalshi": - return await self._kalshi_detail(market_id) - # Limitless and Manifold details on demand - return None - - # ── Polymarket ────────────────────────────────────────── - - async def _search_polymarket(self, query: str) -> list[PredictionMarket]: - """Search Polymarket Gamma API.""" - try: - resp = await self._http.get( - f"{POLYMARKET_GAMMA}/public-search", - params={"q": query}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Polymarket search returned {resp.status_code}") - return [] - - data = resp.json() - events = data.get("events", []) - markets = [] - - for event in events[:10]: - for m in event.get("markets", [])[:5]: - pm = self._parse_polymarket_market(m, event) - if pm: - markets.append(pm) - - return markets - except Exception as e: - logger.warning(f"Polymarket search error: {e}") - return [] - - def _parse_polymarket_market(self, m: dict, event: dict | None = None) -> PredictionMarket | None: - """Parse a Polymarket market dict into unified PredictionMarket.""" - try: - question = m.get("question", "") - slug = m.get("slug", "") - - # Parse double-encoded JSON fields - prices = self._parse_json_field(m.get("outcomePrices", "[]")) - self._parse_json_field(m.get("outcomes", "[]")) - self._parse_json_field(m.get("clobTokenIds", "[]")) - - if isinstance(prices, list) and len(prices) >= 2: - prob_yes = float(prices[0]) - prob_no = float(prices[1]) - else: - prob_yes = 0.5 - prob_no = 0.5 - - volume = float(m.get("volume", 0)) - liquidity = float(m.get("liquidity", 0)) - - # Extract token mentions from question - tokens_mentioned = _extract_token_symbols(question) - - tags = [] - if event: - tags.extend([t.get("label", "") for t in event.get("tags", [])]) - - return PredictionMarket( - source="polymarket", - source_id=slug, - question=question, - slug=slug, - probability_yes=prob_yes, - probability_no=prob_no, - volume_usd=volume, - liquidity_usd=liquidity, - category=m.get("category", event.get("category", "") if event else ""), - tags=tags, - tokens_mentioned=tokens_mentioned, - url=f"https://polymarket.com/event/{slug}" if slug else "", - ends_at=m.get("endDate", event.get("endDate", "") if event else ""), - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Polymarket market: {e}") - return None - - async def _trending_polymarket(self, limit: int) -> list[PredictionMarket]: - """Get trending Polymarket events by volume.""" - try: - resp = await self._http.get( - f"{POLYMARKET_GAMMA}/events", - params={ - "limit": limit, - "active": "true", - "closed": "false", - "order": "volume", - "ascending": "false", - }, - timeout=10.0, - ) - if resp.status_code != 200: - return [] - - events = resp.json() - markets = [] - for event in events[:limit]: - for m in event.get("markets", [])[:3]: - pm = self._parse_polymarket_market(m, event) - if pm: - markets.append(pm) - return markets - except Exception as e: - logger.warning(f"Polymarket trending error: {e}") - return [] - - async def _polymarket_detail(self, slug: str) -> PredictionMarket | None: - """Get detailed Polymarket market data including CLOB prices.""" - try: - # Fetch from Gamma - resp = await self._http.get( - f"{POLYMARKET_GAMMA}/markets", - params={"slug": slug}, - timeout=10.0, - ) - if resp.status_code != 200: - return None - - data = resp.json() - if not data: - return None - - m = data[0] - pm = self._parse_polymarket_market(m) - - # Also fetch CLOB price for live data - if pm: - tokens = self._parse_json_field(m.get("clobTokenIds", "[]")) - if isinstance(tokens, list) and len(tokens) >= 2: - try: - price_resp = await self._http.get( - f"{POLYMARKET_CLOB}/price", - params={"token_id": tokens[0], "side": "buy"}, - timeout=5.0, - ) - if price_resp.status_code == 200: - price_data = price_resp.json() - live_price = float(price_data.get("price", pm.probability_yes)) - pm.probability_yes = live_price - pm.probability_no = 1.0 - live_price - except Exception: - pass # CLOB price is a bonus, Gamma price is fine - - return pm - except Exception as e: - logger.warning(f"Polymarket detail error for {slug}: {e}") - return None - - # ── Kalshi ────────────────────────────────────────────── - - async def _search_kalshi(self, query: str) -> list[PredictionMarket]: - """Search Kalshi by scanning events then fetching their markets.""" - try: - # Step 1: Get open events (organized by category, not sports-dominant) - resp = await self._http.get( - f"{KALSHI_BASE}/events", - params={"status": "open", "limit": 50}, - headers={"Accept": "application/json"}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Kalshi events returned {resp.status_code}") - return [] - - data = resp.json() - events = data.get("events", []) - query_lower = query.lower() - query_terms = query_lower.split() - - results = [] - - # Step 2: Check event titles for matches, then fetch markets - for i, event in enumerate(events[:10]): - event_title = event.get("title", "").lower() - event_ticker = event.get("ticker", "") - - # Match if query terms appear in event title - if not any(term in event_title for term in query_terms): - continue - - # Rate limit: small delay between event fetches - if i > 0: - await asyncio.sleep(0.3) - - # Step 3: Fetch markets for this event - try: - mr = await self._http.get( - f"{KALSHI_BASE}/markets", - params={"event_ticker": event_ticker, "status": "open", "limit": 10}, - headers={"Accept": "application/json"}, - timeout=8.0, - ) - if mr.status_code == 200: - markets_data = mr.json() - for m in markets_data.get("markets", []): - pm = self._parse_kalshi_market(m) - if pm: - results.append(pm) - except Exception: - continue - - return results - except Exception as e: - logger.warning(f"Kalshi search error: {e}") - return [] - - def _parse_kalshi_market(self, m: dict) -> PredictionMarket | None: - """Parse a Kalshi market dict into unified PredictionMarket.""" - try: - ticker = m.get("ticker", "") - title = m.get("title", "") - yes_bid = float(m.get("yes_bid_dollars", 0)) - volume = float(m.get("volume_fp", 0)) # Kalshi uses fake-penny notation - m.get("event_ticker", "") - category = m.get("category", "") - - # Skip multi-outcome markets (sports parlays, etc.) - they have no yes_bid - if yes_bid <= 0 or ",yes " in title.lower(): - return None - - prob_yes = yes_bid # Best YES bid approximates probability - prob_no = 1.0 - prob_yes if prob_yes else 0.5 - - tokens_mentioned = _extract_token_symbols(title) - - return PredictionMarket( - source="kalshi", - source_id=ticker, - question=title, - slug=ticker, - probability_yes=prob_yes, - probability_no=prob_no, - volume_usd=volume, - category=category, - tokens_mentioned=tokens_mentioned, - url=f"https://kalshi.com/markets/{ticker}" if ticker else "", - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Kalshi market: {e}") - return None - - async def _trending_kalshi(self, limit: int) -> list[PredictionMarket]: - """Get trending Kalshi markets by volume - uses events-first approach.""" - try: - # Get open events (avoid sports-multi-outcome noise from raw /markets) - resp = await self._http.get( - f"{KALSHI_BASE}/events", - params={"status": "open", "limit": min(limit * 2, 30)}, - timeout=10.0, - ) - if resp.status_code != 200: - return [] - - data = resp.json() - events = data.get("events", []) - - results = [] - for event in events[:limit]: - event_ticker = event.get("ticker", "") - try: - mr = await self._http.get( - f"{KALSHI_BASE}/markets", - params={"event_ticker": event_ticker, "status": "open", "limit": 5}, - timeout=8.0, - ) - if mr.status_code == 200: - markets_data = mr.json() - for m in markets_data.get("markets", []): - pm = self._parse_kalshi_market(m) - if pm: - results.append(pm) - except Exception: - continue - - return results[:limit] - except Exception as e: - logger.warning(f"Kalshi trending error: {e}") - return [] - - async def _kalshi_detail(self, ticker: str) -> PredictionMarket | None: - """Get detailed Kalshi market data including orderbook.""" - try: - resp = await self._http.get( - f"{KALSHI_BASE}/markets/{ticker}/orderbook", - timeout=10.0, - ) - if resp.status_code != 200: - return None - - data = resp.json() - orderbook = data.get("orderbook_fp", {}) - yes_bids = orderbook.get("yes_dollars", []) - best_yes = float(yes_bids[0][0]) if yes_bids else 0.5 - - # Also get market metadata - meta_resp = await self._http.get( - f"{KALSHI_BASE}/markets", - params={"ticker": ticker}, - timeout=10.0, - ) - if meta_resp.status_code == 200: - meta_data = meta_resp.json() - markets = meta_data.get("markets", []) - if markets: - pm = self._parse_kalshi_market(markets[0]) - if pm: - pm.probability_yes = best_yes - pm.probability_no = 1.0 - best_yes - return pm - - return None - except Exception as e: - logger.warning(f"Kalshi detail error for {ticker}: {e}") - return None - - # ── Limitless ─────────────────────────────────────────── - - async def _search_limitless(self, query: str) -> list[PredictionMarket]: - """Search Limitless Exchange markets by fetching active and filtering.""" - try: - resp = await self._http.get( - f"{LIMITLESS_BASE}/markets/active", - params={"limit": 25}, - headers={"Accept": "application/json"}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Limitless markets returned {resp.status_code}") - return [] - - data = resp.json() - all_markets = data.get("data", []) - - # Filter client-side by query terms - query_lower = query.lower() - query_terms = query_lower.split() - - results = [] - for m in all_markets: - title = m.get("title", "").lower() - if any(term in title for term in query_terms): - pm = self._parse_limitless_market(m) - if pm: - results.append(pm) - - return results - except Exception as e: - logger.warning(f"Limitless search error: {e}") - return [] - - def _parse_limitless_market(self, m: dict) -> PredictionMarket | None: - """Parse a Limitless market dict into unified PredictionMarket.""" - try: - title = m.get("title", "") - slug = m.get("slug", str(m.get("id", ""))) - - # Prices: [YES%, NO%] - e.g., [42.8, 57.2] - prices = m.get("prices", [50, 50]) - prob_yes = float(prices[0]) / 100 if isinstance(prices, list) and len(prices) >= 1 else 0.5 - prob_no = float(prices[1]) / 100 if isinstance(prices, list) and len(prices) >= 2 else 0.5 - - # Volume: use volumeFormatted if available, else volume - vol_str = m.get("volumeFormatted", str(m.get("volume", 0))) - volume = float(vol_str) if vol_str else 0.0 - - categories = m.get("categories", []) - category = categories[0] if categories else "" - tags = m.get("tags", []) - tokens_mentioned = _extract_token_symbols(title) - - return PredictionMarket( - source="limitless", - source_id=str(slug), - question=title, - slug=str(slug), - probability_yes=prob_yes, - probability_no=prob_no, - volume_usd=volume, - category=category, - tags=tags, - tokens_mentioned=tokens_mentioned, - url=f"https://limitless.exchange/markets/{slug}" if slug else "", - ends_at=m.get("expirationDate", ""), - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Limitless market: {e}") - return None - - async def _trending_limitless(self, limit: int) -> list[PredictionMarket]: - """Get trending Limitless markets.""" - try: - limit = min(limit, 25) # API max - resp = await self._http.get( - f"{LIMITLESS_BASE}/markets/active", - params={"limit": limit}, - headers={"Accept": "application/json"}, - timeout=10.0, - ) - if resp.status_code != 200: - return [] - - data = resp.json() - markets = data.get("data", []) - return [pm for m in markets[:limit] if (pm := self._parse_limitless_market(m))] - except Exception as e: - logger.warning(f"Limitless trending error: {e}") - return [] - - # ── Manifold ──────────────────────────────────────────── - - async def _search_manifold(self, query: str) -> list[PredictionMarket]: - """Search Manifold Markets (play-money, sentiment signals). - - Manifold is pure play-money but useful for: - - Forecasting community sentiment - - Early signal detection (top forecasters often move before real-money markets) - - Broad question coverage (more niche crypto questions than Polymarket) - """ - try: - resp = await self._http.get( - f"{MANIFOLD_BASE}/search-markets", - params={"term": query, "limit": 20}, - timeout=10.0, - ) - if resp.status_code != 200: - logger.warning(f"Manifold search returned {resp.status_code}") - return [] - - data = resp.json() - contracts = data if isinstance(data, list) else data.get("contracts", data.get("markets", [])) - - results = [] - for c in contracts[:10]: - pm = self._parse_manifold_market(c) - if pm: - results.append(pm) - - return results - except Exception as e: - logger.warning(f"Manifold search error: {e}") - return [] - - def _parse_manifold_market(self, c: dict) -> PredictionMarket | None: - """Parse a Manifold contract into unified PredictionMarket.""" - try: - question = c.get("question", "") - slug = c.get("slug", c.get("id", "")) - prob = float(c.get("probability", c.get("prob", 0.5))) - volume = float(c.get("volume", c.get("volume24Hours", 0))) - - # Manifold uses "Mana" play money, volume is a signal but lower weight - tokens_mentioned = _extract_token_symbols(question) - tags = list(c.get("tags", [])) - - return PredictionMarket( - source="manifold", - source_id=str(slug), - question=question, - slug=str(slug), - probability_yes=prob, - probability_no=1.0 - prob, - volume_usd=volume, - category="", - tags=tags, - tokens_mentioned=tokens_mentioned, - url=f"https://manifold.markets/{c.get('creatorUsername', '')}/{slug}" if slug else "", - updated_at=datetime.now(UTC).isoformat(), - ) - except Exception as e: - logger.warning(f"Failed to parse Manifold market: {e}") - return None - - # ── Helpers ───────────────────────────────────────────── - - @staticmethod - def _parse_json_field(val): - """Parse double-encoded JSON fields (Polymarket Gamma API).""" - if isinstance(val, str): - try: - return json.loads(val) - except (json.JSONDecodeError, TypeError): - return val - return val - - -# ── Singleton ──────────────────────────────────────────────────── - -_service: PredictionMarketService | None = None - - -def get_prediction_market_service() -> PredictionMarketService: - """Get or create the singleton PredictionMarketService.""" - global _service - if _service is None: - _service = PredictionMarketService() - return _service - - -# ── Helpers: Token Extraction ──────────────────────────────────── - -# Common token symbols to detect in market questions -_COMMON_TOKENS = { - "BTC", - "ETH", - "SOL", - "USDT", - "USDC", - "DAI", - "BNB", - "XRP", - "ADA", - "DOGE", - "MATIC", - "POL", - "DOT", - "AVAX", - "LINK", - "UNI", - "AAVE", - "ARB", - "OP", - "SUI", - "APT", - "TIA", - "SEI", - "STRK", - "WLD", - "PEPE", - "SHIB", - "BONK", - "WIF", - "JUP", - "PYTH", - "RNDR", - "FET", - "AGIX", - "OCEAN", - "IMX", - "INJ", - "EIGEN", - "ENA", - "ETHFI", -} - -# Broader project names often referenced in prediction markets -_COMMON_PROJECTS = { - "polymarket", - "kalshi", - "manifold", - "uniswap", - "sushiswap", - "aave", - "compound", - "makerdao", - "maker", - "lido", - "eigenlayer", - "chainlink", - "arbitrum", - "optimism", - "polygon", - "avalanche", - "fantom", - "near", - "celestia", - "worldcoin", - "tether", - "circle", - "coinbase", - "binance", - "kraken", - "ftx", - "celcius", - "blockfi", - "three arrows", - "alameda", - "jump crypto", - "wintermute", - "curve", - "balancer", - "thorchain", - "osmosis", - "dydx", - "gmx", - "hyperliquid", - "jupiter", - "raydium", - "orca", - "wormhole", - "layerzero", - "zksync", - "starknet", - "scroll", - "linea", - "base", - "mantle", - "mode", - "blast", -} - - -def _extract_token_symbols(text: str) -> list[str]: - """Extract known token symbols and project names from text.""" - found = [] - text_upper = text.upper() - text_lower = text.lower() - - # Check token symbols (typically uppercase in text) - for token in _COMMON_TOKENS: - # Match as word boundary: " BTC " or "BTC's" or "$BTC" - if ( - f" {token} " in f" {text_upper} " - or f"${token}" in text_upper - or text_upper.startswith(f"{token} ") - or text_upper.endswith(f" {token}") - ) and token not in found: - found.append(token) - - # Check project names (case-insensitive) - for project in _COMMON_PROJECTS: - if project in text_lower and project.upper() not in found: - found.append(project) - - return found - - -# ── Caching Helpers ────────────────────────────────────────────── - - -def _cache_hash(*args) -> str: - """Create a short hash for cache keys.""" - raw = "|".join(str(a) for a in args) - return hashlib.md5(raw.encode()).hexdigest()[:12] - - -def _market_to_dict(m: PredictionMarket) -> dict: - """Serialize PredictionMarket to dict for JSON caching.""" - return { - "source": m.source, - "source_id": m.source_id, - "question": m.question, - "slug": m.slug, - "probability_yes": m.probability_yes, - "probability_no": m.probability_no, - "volume_usd": m.volume_usd, - "liquidity_usd": m.liquidity_usd, - "category": m.category, - "tags": m.tags, - "tokens_mentioned": m.tokens_mentioned, - "is_security_relevant": m.is_security_relevant, - "is_crypto_relevant": m.is_crypto_relevant, - "url": m.url, - "ends_at": m.ends_at, - "updated_at": m.updated_at, - } - - -def _dict_to_market(d: dict) -> PredictionMarket: - """Deserialize dict back to PredictionMarket.""" - return PredictionMarket( - source=d.get("source", ""), - source_id=d.get("source_id", ""), - question=d.get("question", ""), - slug=d.get("slug", ""), - probability_yes=float(d.get("probability_yes", 0.5)), - probability_no=float(d.get("probability_no", 0.5)), - volume_usd=float(d.get("volume_usd", 0)), - liquidity_usd=float(d.get("liquidity_usd", 0)), - category=d.get("category", ""), - tags=d.get("tags", []), - tokens_mentioned=d.get("tokens_mentioned", []), - is_security_relevant=d.get("is_security_relevant", False), - is_crypto_relevant=d.get("is_crypto_relevant", False), - url=d.get("url", ""), - ends_at=d.get("ends_at", ""), - updated_at=d.get("updated_at", ""), - ) - - -def _digest_to_dict(d: PredictionDigest) -> dict: - """Serialize PredictionDigest to dict for JSON caching.""" - return { - "generated_at": d.generated_at, - "total_markets_searched": d.total_markets_searched, - "security_relevant_count": d.security_relevant_count, - "crypto_relevant_count": d.crypto_relevant_count, - "top_threats": [_market_to_dict(m) for m in d.top_threats], - "token_specific_markets": [_market_to_dict(m) for m in d.token_specific_markets], - "ecosystem_risk_markets": [_market_to_dict(m) for m in d.ecosystem_risk_markets], - "regulatory_markets": [_market_to_dict(m) for m in d.regulatory_markets], - } - - -def _dict_to_digest(d: dict) -> PredictionDigest: - """Deserialize dict back to PredictionDigest.""" - return PredictionDigest( - generated_at=d.get("generated_at", ""), - total_markets_searched=d.get("total_markets_searched", 0), - security_relevant_count=d.get("security_relevant_count", 0), - crypto_relevant_count=d.get("crypto_relevant_count", 0), - top_threats=[_dict_to_market(m) for m in d.get("top_threats", [])], - token_specific_markets=[_dict_to_market(m) for m in d.get("token_specific_markets", [])], - ecosystem_risk_markets=[_dict_to_market(m) for m in d.get("ecosystem_risk_markets", [])], - regulatory_markets=[_dict_to_market(m) for m in d.get("regulatory_markets", [])], - ) +from __future__ import annotations + +from app.domains.markets import ( # noqa: F401 + PredictionDigest, + PredictionMarket, + PredictionMarketService, + get_prediction_market_service, +) diff --git a/app/routers/admin_backend.py b/app/routers/admin_backend.py index e5b662d..e197629 100644 --- a/app/routers/admin_backend.py +++ b/app/routers/admin_backend.py @@ -28,7 +28,7 @@ from datetime import datetime, timedelta from fastapi import APIRouter, Body, HTTPException, Request from pydantic import BaseModel, Field -from app.admin_backend import ( +from app.domains.admin import ( PERMISSIONS, AdminRole, AdminUserStore, diff --git a/app/routers/analytics.py b/app/routers/analytics.py index d8ff21c..6153f35 100644 --- a/app/routers/analytics.py +++ b/app/routers/analytics.py @@ -25,8 +25,8 @@ from typing import Any from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field -from app.admin_backend import AuditLogger, require_admin from app.analytics_engine import AnalyticsEngine, DashboardWidget, get_analytics_engine +from app.domains.admin import AuditLogger, require_admin logger = logging.getLogger("analytics_api") diff --git a/app/routers/bulletin_board.py b/app/routers/bulletin_board.py index f8341e8..afd96a2 100644 --- a/app/routers/bulletin_board.py +++ b/app/routers/bulletin_board.py @@ -25,8 +25,8 @@ Admin endpoints (require admin session): from fastapi import APIRouter, Body, HTTPException, Request from pydantic import BaseModel, Field -from app.admin_backend import AuditLogger, require_admin -from app.bulletin_board import ( +from app.domains.admin import AuditLogger, require_admin +from app.domains.bulletin import ( BulletinBoardManager, PostCategory, PostStatus, @@ -576,7 +576,7 @@ async def admin_moderate_comment( @router.get("/badges/{user_id}") async def get_badges(user_id: str): """Get user badges.""" - from app.bulletin_board import get_user_badges, get_user_reputation + from app.domains.bulletin import get_user_badges, get_user_reputation badges = await get_user_badges(user_id) rep = await get_user_reputation(user_id) @@ -613,7 +613,7 @@ async def x402_bot_post(request: Request, data: dict): Bot posting via x402 - $1 TO JOIN (one-time, 30-day membership). Pay $1 once, post unlimited for 30 days. Includes badges. """ - from app.bulletin_board import verify_x402_bot + from app.domains.bulletin import verify_x402_bot bot_addr = data.get("bot_address", "") tx_hash = data.get("tx_hash", "") diff --git a/app/routers/prediction_market_router.py b/app/routers/prediction_market_router.py index 37ebf7b..d5b4201 100644 --- a/app/routers/prediction_market_router.py +++ b/app/routers/prediction_market_router.py @@ -23,7 +23,7 @@ import logging from fastapi import APIRouter, HTTPException, Query -from app.prediction_market_service import ( +from app.domains.markets import ( PredictionDigest, PredictionMarket, get_prediction_market_service, diff --git a/app/routers/wallet_manager_v2.py b/app/routers/wallet_manager_v2.py index 869f5ef..c97ff16 100644 --- a/app/routers/wallet_manager_v2.py +++ b/app/routers/wallet_manager_v2.py @@ -33,7 +33,7 @@ from typing import Any from fastapi import APIRouter, Body, HTTPException, Request from pydantic import BaseModel, Field -from app.admin_backend import AuditLogger, require_admin +from app.domains.admin import AuditLogger, require_admin from app.wallet_manager_v2 import ( CHAIN_REGISTRY, PaymentRecord, diff --git a/app/routers/x402_mcp_handler.py b/app/routers/x402_mcp_handler.py index a63d55f..1832ac1 100644 --- a/app/routers/x402_mcp_handler.py +++ b/app/routers/x402_mcp_handler.py @@ -10,7 +10,7 @@ import os from fastapi import APIRouter, HTTPException, Request -from app.mcp.x402_mcp_server import TOOLS, handle_mcp_call +from app.domains.mcp.x402_mcp_server import TOOLS, handle_mcp_call router = APIRouter(tags=["x402-MCP"]) diff --git a/app/security_defense.py b/app/security_defense.py index 630d46b..c43f3e3 100644 --- a/app/security_defense.py +++ b/app/security_defense.py @@ -526,7 +526,7 @@ class HoneypotSystem: await BotDetectionEngine._log_security_event(event) # Auto-ban the IP - from app.admin_backend import SecurityManager + from app.domains.admin import SecurityManager await SecurityManager.block_ip(ip, f"Honeypot triggered: {path}", 168) # 7 days diff --git a/app/services/prediction_market_intel.py b/app/services/prediction_market_intel.py index a5c17e8..5cb4332 100644 --- a/app/services/prediction_market_intel.py +++ b/app/services/prediction_market_intel.py @@ -27,7 +27,7 @@ import logging from dataclasses import dataclass, field from datetime import UTC, datetime -from app.prediction_market_service import ( +from app.domains.markets import ( PredictionMarket, get_prediction_market_service, ) diff --git a/mypy-gate.ini b/mypy-gate.ini new file mode 100644 index 0000000..340ec5f --- /dev/null +++ b/mypy-gate.ini @@ -0,0 +1,14 @@ +[mypy] +strict = true +ignore_missing_imports = true +explicit_package_bases = true +exclude = (\.venv/|tests/|__pycache__/) + +[mypy-app.core.redis] +ignore_errors = false + +[mypy-app.domains.auth.*] +ignore_errors = false + +[mypy-*] +ignore_errors = true diff --git a/mypy.ini b/mypy.ini index efdbb65..8ae7d38 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,5 @@ [mypy] strict = true ignore_missing_imports = true +explicit_package_bases = true exclude = (\.venv/|tests/|__pycache__/) diff --git a/tests/unit/test_tier1_moat_mcp.py b/tests/unit/test_tier1_moat_mcp.py index 8675f50..b20363f 100644 --- a/tests/unit/test_tier1_moat_mcp.py +++ b/tests/unit/test_tier1_moat_mcp.py @@ -5,7 +5,7 @@ needing a live database or external service. They test the contract. """ from __future__ import annotations -from app.mcp.server import ( +from app.domains.mcp.server import ( MCP_PROTOCOL_VERSION, MCP_SERVER_VERSION, TOOL_CATALOG, diff --git a/tests/unit/test_tier2_eth_labels_mcp.py b/tests/unit/test_tier2_eth_labels_mcp.py index 0fcc24f..676b479 100644 --- a/tests/unit/test_tier2_eth_labels_mcp.py +++ b/tests/unit/test_tier2_eth_labels_mcp.py @@ -9,15 +9,15 @@ from unittest.mock import patch import pytest -from app.mcp.server import call_tool +from app.domains.mcp.server import call_tool @pytest.mark.asyncio async def test_eth_labels_query_tool_exists(): """Test that eth_labels_query tool is registered in the server.""" - import app.mcp.server + import app.domains.mcp.server # Check if it's in the catalog - tool_names = [tool["name"] for tool in app.mcp.server.TOOL_CATALOG] + tool_names = [tool["name"] for tool in app.domains.mcp.server.TOOL_CATALOG] assert "eth_labels_query" in tool_names print("✓ eth_labels_query found in tool catalog") @@ -25,8 +25,8 @@ async def test_eth_labels_query_tool_exists(): @pytest.mark.asyncio async def test_eth_labels_stats_tool_exists(): """Test that eth_labels_stats tool is registered in the server.""" - import app.mcp.server - tool_names = [tool["name"] for tool in app.mcp.server.TOOL_CATALOG] + import app.domains.mcp.server + tool_names = [tool["name"] for tool in app.domains.mcp.server.TOOL_CATALOG] assert "eth_labels_stats" in tool_names print("✓ eth_labels_stats found in tool catalog") @@ -34,7 +34,7 @@ async def test_eth_labels_stats_tool_exists(): @pytest.mark.asyncio async def test_eth_labels_query_version_tracked(): """Test that eth_labels_query tool has version tracking.""" - from app.mcp.server import TOOL_VERSIONS + from app.domains.mcp.server import TOOL_VERSIONS assert "eth_labels_query" in TOOL_VERSIONS version = TOOL_VERSIONS["eth_labels_query"] assert version == "1.0.0" # Version from Tier 2 @@ -44,7 +44,7 @@ async def test_eth_labels_query_version_tracked(): @pytest.mark.asyncio async def test_eth_labels_stats_version_tracked(): """Test that eth_labels_stats tool has version tracking.""" - from app.mcp.server import TOOL_VERSIONS + from app.domains.mcp.server import TOOL_VERSIONS assert "eth_labels_stats" in TOOL_VERSIONS version = TOOL_VERSIONS["eth_labels_stats"] assert version == "1.0.0" # Version from Tier 2 @@ -52,7 +52,7 @@ async def test_eth_labels_stats_version_tracked(): @pytest.mark.asyncio -@patch('app.mcp.tools.eth_labels_tool.query_eth_labels_db_mcp') +@patch('app.domains.mcp.tools.eth_labels_tool.query_eth_labels_db_mcp') async def test_eth_labels_query_calls_underlying_function(mock_query): """Test that eth_labels_query tool calls our implementation.""" # Mock the response @@ -71,7 +71,7 @@ async def test_eth_labels_query_calls_underlying_function(mock_query): @pytest.mark.asyncio -@patch('app.mcp.tools.eth_labels_tool.get_eth_labels_stats_mcp') +@patch('app.domains.mcp.tools.eth_labels_tool.get_eth_labels_stats_mcp') async def test_eth_labels_stats_calls_underlying_function(mock_stats): """Test that eth_labels_stats calls our implementation.""" mock_response = {"total_accounts": 106000, "tables": ["accounts"]} From 1b5333269523d65b99060197977e82a38c804ae7 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 17:44:09 +0700 Subject: [PATCH 11/12] fix(auth): mount auth router + rename __init__ to router.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1.1 — Implement real ai_router.chat_completion / stream_chat_completion using Ollama Cloud primary + OpenRouter fallback. Fixes AttributeError on /api/v1/chat and /api/v1/wallet-clusters. P1.2 — Fix broken RAG call paths: telegram bot now hits /api/v1/rag/v2/search then ai_router.chat_completion; unified_wallet_scanner._rag_enrich uses async httpx + correct path. Mount app.domains.databus.router. P1.3 — Implement setup_otel / shutdown_otel in core/tracing.py and create core/langfuse.py with init_langfuse / flush_langfuse as safe no-ops when SDK is missing or env vars unset. P1.4 — Fix Prometheus alert rule metric name drift: rmi_requests_total -> rmi_http_requests_total (matches metrics.py). P1.5 — Move dead admin_backend.py (1691 LOC) to app/domains/admin/router.py and mount it. Admin login endpoint now reachable. P2.1 — Rename app/domains/auth/__init__.py (507 LOC router file that broke mypy) -> app/domains/auth/router.py. Mount app.domains.auth.router so /register, /login, /wallet/*, /user/me, /2fa/*, OAuth, Telegram are live. P2.1b — Add pyotp + qrcode to requirements.txt so 2FA endpoints don't 503. --- app/ai_router.py | 286 ++++++++-- app/core/langfuse.py | 49 ++ app/core/tracing.py | 44 ++ .../admin/router.py} | 0 app/domains/auth/__init__.py | 532 +++--------------- app/domains/auth/router.py | 509 +++++++++++++++++ app/domains/mcp/__init__.py | 4 +- app/domains/telegram/rugmunchbot/bot.py | 37 +- app/mcp/__init__.py | 6 +- app/mount.py | 3 + app/routers/chat.py | 2 +- app/routers/wallet_clustering_router.py | 8 +- app/unified_wallet_scanner.py | 35 +- infra/alert_rules.yml | 12 +- requirements.txt | 2 + 15 files changed, 992 insertions(+), 537 deletions(-) create mode 100644 app/core/langfuse.py rename app/{routers/admin_backend.py => domains/admin/router.py} (100%) create mode 100644 app/domains/auth/router.py diff --git a/app/ai_router.py b/app/ai_router.py index 5fccd1d..621f592 100644 --- a/app/ai_router.py +++ b/app/ai_router.py @@ -1,72 +1,288 @@ #!/usr/bin/env python3 """ -Stub AI Router - Intelligent Model-First Provider Swapping -=========================================================== -Routes requests to optimal AI provider based on quota, latency, and cost. -For now, a minimal stub that delegates to OpenRouter. +AI Router — service functions for chat completions. + +Primary: Ollama Cloud (deepseek-v4-flash, free). +Fallback: OpenRouter (various models via OPENROUTER_API_KEY). + +Both endpoints are OpenAI-compatible /v1/chat/completions. +Called by app/routers/chat.py and app/routers/wallet_clustering_router.py +as ``ai_router.chat_completion(...)`` and +``ai_router.stream_chat_completion(...)``. """ import base64 +import json +import logging import os +import time +from collections.abc import AsyncGenerator from typing import Any +import httpx from fastapi import APIRouter +logger = logging.getLogger("rmi.ai_router") + router = APIRouter(tags=["AI Router"]) # Decode base64 LLM key if present, otherwise use plain LLM_API_KEY -# (safety net: ensures key is decoded even when imported without main.py) if os.getenv("LLM_API_KEY_B64"): os.environ["LLM_API_KEY"] = base64.b64decode(os.getenv("LLM_API_KEY_B64")).decode() -# Model tiers (for reference, full config in ai_router.py) -MODEL_TIERS = { - "T0": {"name": "Ultra", "models": ["gpt-4o", "claude-3.5-sonnet"], "max_cost_per_1k": 0.05}, - "T1": {"name": "Premium", "models": ["gpt-4-turbo", "claude-3-opus"], "max_cost_per_1k": 0.02}, - "T2": { - "name": "Standard", - "models": ["gpt-3.5-turbo", "claude-3-haiku"], - "max_cost_per_1k": 0.005, - }, - "T3": {"name": "Fast", "models": ["llama-3-8b", "mistral-tiny"], "max_cost_per_1k": 0.001}, - "T4": {"name": "Free", "models": ["tiny-llama", "phi-2"], "max_cost_per_1k": 0.0}, +# ── Provider config ───────────────────────────────────────────── +OLLAMA_KEY = os.getenv("OLLAMA_API_KEY", "") +OLLAMA_URL = "https://ollama.com/v1/chat/completions" +OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "deepseek-v4-flash") + +OPENROUTER_KEY = os.getenv("OPENROUTER_API_KEY", "") +OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" +OPENROUTER_MODEL = os.getenv("OPENROUTER_MODEL", "deepseek/deepseek-chat") + +# Tier → model mapping (best effort; Ollama Cloud ignores non-model fields) +TIER_MODELS: dict[str, str] = { + "T0": "deepseek-v4-pro", + "T1": "deepseek-v4-pro", + "T2": "deepseek-v4-flash", + "T3": "deepseek-v4-flash", + "T4": "deepseek-v4-flash", } -# Providers (for reference) -PROVIDERS = { - "deepseek": { - "url": os.getenv("LLM_BASE_URL", "https://api.deepseek.com/v1/chat/completions"), - "key_env": "LLM_API_KEY", - "model": os.getenv("LLM_MODEL", "deepseek-v4-flash"), - "rpm": 100, - }, - "openrouter": { - "url": "https://openrouter.ai/api/v1/chat/completions", - "key_env": "OPENROUTER_API_KEY", - "rpm": 100, - }, +MODEL_TIERS = { + "T0": {"name": "Ultra", "models": ["deepseek-v4-pro"], "max_cost_per_1k": 0.0}, + "T1": {"name": "Premium", "models": ["deepseek-v4-pro"], "max_cost_per_1k": 0.0}, + "T2": {"name": "Standard", "models": ["deepseek-v4-flash"], "max_cost_per_1k": 0.0}, + "T3": {"name": "Fast", "models": ["deepseek-v4-flash"], "max_cost_per_1k": 0.0}, + "T4": {"name": "Free", "models": ["deepseek-v4-flash"], "max_cost_per_1k": 0.0}, } +PROVIDERS = { + "ollama": {"url": OLLAMA_URL, "key_env": "OLLAMA_API_KEY", "model": OLLAMA_MODEL, "rpm": 100}, + "openrouter": {"url": OPENROUTER_URL, "key_env": "OPENROUTER_API_KEY", "model": OPENROUTER_MODEL, "rpm": 100}, +} + + +def _resolve_model(model: str | None, tier: str | None) -> str: + """Pick the model: explicit model > tier mapping > default.""" + if model: + return model + if tier and tier in TIER_MODELS: + return TIER_MODELS[tier] + return OLLAMA_MODEL + + +async def chat_completion( + *, + messages: list[dict[str, str]], + model: str | None = None, + tier: str | None = None, + temperature: float = 0.3, + max_tokens: int = 500, + timeout: float = 30.0, +) -> dict[str, Any]: + """Non-streaming chat completion. + + Returns dict with keys: content, model, _provider, _latency_ms, usage, error (on failure). + Tries Ollama Cloud first, falls back to OpenRouter. + """ + resolved = _resolve_model(model, tier) + start = time.time() + + # Provider 1: Ollama Cloud + if OLLAMA_KEY: + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post( + OLLAMA_URL, + headers={ + "Authorization": f"Bearer {OLLAMA_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": resolved, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": False, + }, + ) + if resp.status_code == 200: + data = resp.json() + content = data.get("choices", [{}])[0].get("message", {}).get("content", "") + return { + "content": content, + "model": data.get("model", resolved), + "_provider": "ollama", + "_latency_ms": round((time.time() - start) * 1000, 1), + "usage": data.get("usage", {}), + } + logger.warning("ai_router ollama status=%s body=%s", resp.status_code, resp.text[:200]) + except Exception as e: + logger.warning("ai_router ollama error: %s", e) + + # Provider 2: OpenRouter fallback + if OPENROUTER_KEY: + try: + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.post( + OPENROUTER_URL, + headers={ + "Authorization": f"Bearer {OPENROUTER_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": resolved, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": False, + }, + ) + if resp.status_code == 200: + data = resp.json() + content = data.get("choices", [{}])[0].get("message", {}).get("content", "") + return { + "content": content, + "model": data.get("model", resolved), + "_provider": "openrouter", + "_latency_ms": round((time.time() - start) * 1000, 1), + "usage": data.get("usage", {}), + } + logger.warning("ai_router openrouter status=%s body=%s", resp.status_code, resp.text[:200]) + except Exception as e: + logger.warning("ai_router openrouter error: %s", e) + + return { + "content": "", + "model": resolved, + "_provider": "none", + "_latency_ms": round((time.time() - start) * 1000, 1), + "usage": {}, + "error": "No AI provider available (set OLLAMA_API_KEY or OPENROUTER_API_KEY)", + } + + +async def stream_chat_completion( + *, + messages: list[dict[str, str]], + model: str | None = None, + tier: str | None = None, + temperature: float = 0.3, + max_tokens: int = 500, + timeout: float = 30.0, +) -> AsyncGenerator[str, None]: + """Streaming chat completion via SSE. + + Yields content tokens as strings. On error, yields ``[ERROR: ...]``. + Tries Ollama Cloud first, falls back to OpenRouter. + """ + resolved = _resolve_model(model, tier) + + # Provider 1: Ollama Cloud (streaming) + if OLLAMA_KEY: + try: + async with httpx.AsyncClient(timeout=timeout) as client, client.stream( + "POST", + OLLAMA_URL, + headers={ + "Authorization": f"Bearer {OLLAMA_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": resolved, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + }, + ) as resp: + if resp.status_code == 200: + async for line in resp.aiter_lines(): + if line.startswith("data: ") and line != "data: [DONE]": + try: + chunk = json.loads(line[6:]) + delta = chunk.get("choices", [{}])[0].get("delta", {}) + content = delta.get("content", "") + if content: + yield content + except json.JSONDecodeError: + pass + return + logger.warning("ai_router ollama stream status=%s", resp.status_code) + except Exception as e: + logger.warning("ai_router ollama stream error: %s", e) + + # Provider 2: OpenRouter streaming fallback + if OPENROUTER_KEY: + try: + async with httpx.AsyncClient(timeout=timeout) as client, client.stream( + "POST", + OPENROUTER_URL, + headers={ + "Authorization": f"Bearer {OPENROUTER_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": resolved, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + }, + ) as resp: + if resp.status_code == 200: + async for line in resp.aiter_lines(): + if line.startswith("data: ") and line != "data: [DONE]": + try: + chunk = json.loads(line[6:]) + delta = chunk.get("choices", [{}])[0].get("delta", {}) + content = delta.get("content", "") + if content: + yield content + except json.JSONDecodeError: + pass + return + logger.warning("ai_router openrouter stream status=%s", resp.status_code) + except Exception as e: + logger.warning("ai_router openrouter stream error: %s", e) + + yield "[ERROR: No AI provider available (set OLLAMA_API_KEY or OPENROUTER_API_KEY)]" + + +# ── HTTP endpoints (kept for compat, now backed by real service) ── + @router.post("/ai/completions") -async def ai_completions(request: dict[str, Any]): +async def ai_completions(request: dict[str, Any]) -> dict[str, Any]: """AI completion via optimal provider routing.""" - return {"error": "AI Router not fully configured", "provider": "openrouter"} + return await chat_completion( + messages=request.get("messages", []), + model=request.get("model"), + tier=request.get("tier"), + temperature=request.get("temperature", 0.3), + max_tokens=request.get("max_tokens", 500), + ) @router.post("/ai/chat") -async def ai_chat(request: dict[str, Any]): +async def ai_chat(request: dict[str, Any]) -> dict[str, Any]: """AI chat endpoint with provider fallback.""" - return {"error": "AI Router not fully configured", "provider": "openrouter"} + return await chat_completion( + messages=request.get("messages", []), + model=request.get("model"), + tier=request.get("tier"), + temperature=request.get("temperature", 0.3), + max_tokens=request.get("max_tokens", 500), + ) @router.get("/ai/providers") -async def list_providers(): +async def list_providers() -> dict[str, Any]: """List available AI providers.""" return {"providers": list(PROVIDERS.keys())} @router.get("/ai/models") -async def list_models(): +async def list_models() -> dict[str, Any]: """List available models by tier.""" return {"tiers": MODEL_TIERS} diff --git a/app/core/langfuse.py b/app/core/langfuse.py new file mode 100644 index 0000000..7deec30 --- /dev/null +++ b/app/core/langfuse.py @@ -0,0 +1,49 @@ +"""Langfuse LLM tracing - opt-in via env vars. + +Safe no-op when LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY are unset +or the `langfuse` package is not installed. +""" + +import os + +LANGFUSE_ENABLED = bool( + os.getenv("LANGFUSE_PUBLIC_KEY", "") and os.getenv("LANGFUSE_SECRET_KEY", "") +) +LANGFUSE_HOST = os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com") + +_initialized = False + + +def init_langfuse() -> bool: + """Initialize the Langfuse client. Returns True on success.""" + global _initialized + if not LANGFUSE_ENABLED: + return False + try: + from langfuse import Langfuse # noqa: F401 + + Langfuse( + public_key=os.getenv("LANGFUSE_PUBLIC_KEY"), + secret_key=os.getenv("LANGFUSE_SECRET_KEY"), + host=LANGFUSE_HOST, + ) + _initialized = True + return True + except ImportError: + return False + except Exception: + return False + + +def flush_langfuse() -> None: + """Flush any pending Langfuse events. Safe to call on shutdown.""" + if not _initialized: + return + try: + from langfuse import Langfuse + + Langfuse().flush() + except ImportError: + pass + except Exception: + pass \ No newline at end of file diff --git a/app/core/tracing.py b/app/core/tracing.py index a82d3b7..15e0cd4 100644 --- a/app/core/tracing.py +++ b/app/core/tracing.py @@ -7,6 +7,7 @@ import uuid from fastapi import FastAPI, Request TRACING_ENABLED = os.getenv("OTEL_ENABLED", "false").lower() == "true" +OTEL_ENDPOINT = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") def setup_tracing(app: FastAPI) -> list: @@ -33,6 +34,49 @@ def setup_tracing(app: FastAPI) -> list: return {"traces": [], "note": "OpenTelemetry export to Grafana Tempo configured. Traces available at :3000."} +def setup_otel() -> bool: + """Initialize OpenTelemetry tracing. Returns True on success. + + No-op if OTEL_ENABLED != "true". If OTEL_ENABLED is true but the SDK + is not installed, logs and returns False. + """ + if not TRACING_ENABLED: + return False + try: + # Lazy import — opentelemetry is an optional dependency. + from opentelemetry import trace # noqa: F401 + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter # noqa: F401 + from opentelemetry.sdk.resources import Resource # noqa: F401 + from opentelemetry.sdk.trace import TracerProvider # noqa: F401 + from opentelemetry.sdk.trace.export import BatchSpanProcessor # noqa: F401 + + resource = Resource.create({"service.name": "rmi-backend"}) + provider = TracerProvider(resource=resource) + if OTEL_ENDPOINT: + exporter = OTLPSpanExporter(endpoint=OTEL_ENDPOINT) + provider.add_span_processor(BatchSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + return True + except ImportError: + return False + except Exception: + return False + + +def shutdown_otel() -> None: + """Flush and shut down OpenTelemetry tracer provider.""" + try: + from opentelemetry import trace + + provider = trace.get_tracer_provider() + if hasattr(provider, "shutdown"): + provider.shutdown() + except ImportError: + pass + except Exception: + pass + + def start_span(name: str, attributes: dict | None = None) -> dict: return {"name": name, "start": time.perf_counter(), "attrs": attributes or {}} diff --git a/app/routers/admin_backend.py b/app/domains/admin/router.py similarity index 100% rename from app/routers/admin_backend.py rename to app/domains/admin/router.py diff --git a/app/domains/auth/__init__.py b/app/domains/auth/__init__.py index deb69f7..f4b5687 100644 --- a/app/domains/auth/__init__.py +++ b/app/domains/auth/__init__.py @@ -1,23 +1,21 @@ +"""Auth domain - public API. + +Phase 4 domain consolidation + Phase 2.1 structural fix: +``app/domains/auth/__init__.py`` was a 507-LOC router file that broke mypy +(source file found twice under different module names). The router now +lives in ``app/domains/auth/router.py`` and this package marker only +re-exports the public surface. """ -Auth Router - Complete authentication system (email, wallet, OAuth, Telegram) -""" +from __future__ import annotations -import json -import logging -from datetime import datetime, timedelta -from typing import Any, cast - -from fastapi import APIRouter, HTTPException, Request - -from app.core.redis import get_redis from app.domains.auth.deps import ( get_current_user, - require_auth as require_auth, - require_public_profile as require_public_profile, + require_auth, + require_public_profile, ) from app.domains.auth.jwt import ( JWT_ALGORITHM, - JWT_EXPIRY_DAYS as JWT_EXPIRY_DAYS, + JWT_EXPIRY_DAYS, JWT_SECRET, _create_jwt, _derive_user_id, @@ -26,15 +24,16 @@ from app.domains.auth.jwt import ( ) from app.domains.auth.oauth import router as oauth_router from app.domains.auth.passwords import hash_password, verify_password +from app.domains.auth.router import router from app.domains.auth.schemas import ( EmailLoginRequest, EmailRegisterRequest, - GoogleAuthResponse as GoogleAuthResponse, + GoogleAuthResponse, NonceResponse, - TelegramAuthRequest as TelegramAuthRequest, + TelegramAuthRequest, TwoFAEnableRequest, TwoFALoginRequest, - TwoFASetupResponse as TwoFASetupResponse, + TwoFASetupResponse, TwoFAStatusResponse, TwoFAVerifyRequest, UserResponse, @@ -43,7 +42,7 @@ from app.domains.auth.schemas import ( WalletVerifyRequest, ) from app.domains.auth.store import ( - _delete_user as _delete_user, + _delete_user, _get_user, _get_user_by_email, _is_valid_email, @@ -51,11 +50,11 @@ from app.domains.auth.store import ( ) from app.domains.auth.totp import ( PYOTP_AVAILABLE, - TOTP_DIGITS as TOTP_DIGITS, - TOTP_INTERVAL as TOTP_INTERVAL, - TOTP_ISSUER as TOTP_ISSUER, - TOTP_WINDOW as TOTP_WINDOW, - _build_totp as _build_totp, + TOTP_DIGITS, + TOTP_INTERVAL, + TOTP_ISSUER, + TOTP_WINDOW, + _build_totp, _generate_backup_codes, _generate_qr_base64, _generate_totp_secret, @@ -65,443 +64,52 @@ from app.domains.auth.totp import ( _verify_totp, ) -logger = logging.getLogger(__name__) - -router = APIRouter(tags=["auth"]) - -# Include OAuth/Telegram routes from dedicated submodule -router.include_router(oauth_router, tags=["auth"]) - -# ── Config ── - - -# ── Storage (Redis-backed user store) ── -# ── Email Auth ── -@router.post("/register", response_model=WalletAuthResponse) -def register_email(req: EmailRegisterRequest) -> WalletAuthResponse: - """Register a new user with email/password.""" - if not _is_valid_email(req.email): - raise HTTPException(status_code=400, detail="Invalid email format") - - if len(req.password) < 8: - raise HTTPException(status_code=400, detail="Password must be at least 8 characters") - - # Check if user exists - existing = _get_user_by_email(req.email) - if existing: - raise HTTPException(status_code=400, detail="Email already registered") - - user_id = _derive_user_id(req.email) - hashed = hash_password(req.password) - - user: dict[str, Any] = { - "id": user_id, - "email": req.email, - "display_name": req.display_name, - "password_hash": hashed, - "wallet_address": req.wallet_address, - "wallet_chain": req.wallet_chain, - "tier": "FREE", - "role": "USER", - "created_at": datetime.utcnow().isoformat(), - "xp": 0, - "level": 1, - "badges": [], - "scans_remaining": 5, - "scans_used": 0, - } - - # Save user + email index - r = get_redis() - assert r is not None - r.hset("rmi:users", user_id, json.dumps(user)) - r.hset("rmi:users:email", req.email.lower(), user_id) - - token = _create_jwt(user_id, req.email, "FREE", "USER", req.wallet_address) - - return cast( - "WalletAuthResponse", - { - "access_token": token, - "refresh_token": token, - "user": { - "id": user_id, - "email": req.email, - "display_name": req.display_name, - "wallet_address": req.wallet_address, - "wallet_chain": req.wallet_chain, - "tier": "FREE", - "role": "USER", - "created_at": user["created_at"], - }, - }, - ) - - -@router.post("/login", response_model=WalletAuthResponse) -def login_email(req: EmailLoginRequest) -> WalletAuthResponse: - """Login with email/password. If 2FA enabled, returns partial token requiring /2fa/login.""" - user = _get_user_by_email(req.email) - if not user or not user.get("password_hash"): - raise HTTPException(status_code=401, detail="Invalid credentials") - - if not verify_password(req.password, user["password_hash"]): - raise HTTPException(status_code=401, detail="Invalid credentials") - - # If 2FA enabled, return a pre-auth token that requires TOTP verification - if user.get("totp_enabled") and user.get("totp_secret"): - # Generate a short-lived pre-auth token (5 min) - pre_token = _create_jwt( - user["id"], - user["email"], - user.get("tier", "FREE"), - user.get("role", "USER"), - user.get("wallet_address"), - ) - # Override expiry to 5 minutes - import jose.jwt as jose_jwt - - payload = jose_jwt.decode(pre_token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) - payload["exp"] = datetime.utcnow() + timedelta(minutes=5) - payload["2fa_required"] = True - payload["pre_auth"] = True - pre_token = jose_jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) - - return cast( - "WalletAuthResponse", - { - "access_token": pre_token, - "refresh_token": pre_token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - "_2fa_required": True, - }, - }, - ) - - token = _create_jwt( - user["id"], - user["email"], - user.get("tier", "FREE"), - user.get("role", "USER"), - user.get("wallet_address"), - ) - - return cast( - "WalletAuthResponse", - { - "access_token": token, - "refresh_token": token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - }, - }, - ) - - -# ── Wallet Auth ── -@router.post("/wallet/nonce", response_model=NonceResponse) -async def wallet_nonce(req: WalletNonceRequest) -> NonceResponse: - """Get a nonce for wallet signature.""" - nonce = generate_nonce() - timestamp = datetime.utcnow().isoformat() - message = f"RugMunch Intelligence wants you to sign in with your {req.chain.title()} account.\n\nWallet: {req.address}\nNonce: {nonce}\nTimestamp: {timestamp}" - return cast("NonceResponse", {"nonce": nonce, "timestamp": timestamp, "message": message}) - - -@router.post("/wallet/verify", response_model=WalletAuthResponse) -async def wallet_verify(req: WalletVerifyRequest) -> WalletAuthResponse: - """Verify wallet signature and create session.""" - from app.domains.auth.wallet import get_or_create_wallet_user, verify_wallet_signature - - if not verify_wallet_signature(req.message, req.signature, req.address): - raise HTTPException(status_code=401, detail="Invalid signature") - - user_data = await get_or_create_wallet_user(req.address) - - return cast( - "WalletAuthResponse", - { - "access_token": user_data["access_token"], - "refresh_token": user_data["refresh_token"], - "user": { - "id": user_data["id"], - "email": user_data["email"], - "display_name": user_data.get("display_name", f"Agent {req.address[2:8].upper()}"), - "wallet_address": req.address, - "wallet_chain": req.chain, - "tier": user_data["tier"], - "role": user_data["role"], - "created_at": user_data["created_at"], - }, - }, - ) - - -# ── User Profile ── -@router.get("/user/me", response_model=UserResponse) -async def user_me(request: Request) -> UserResponse: - """Get current authenticated user profile.""" - auth_header = request.headers.get("Authorization", "") - if not auth_header.startswith("Bearer "): - raise HTTPException(status_code=401, detail="Missing auth token") - - token = auth_header[7:] - payload = _verify_jwt(token) - if not payload: - raise HTTPException(status_code=401, detail="Invalid token") - - user = _get_user(payload["id"]) - if not user: - raise HTTPException(status_code=404, detail="User not found") - - return cast( - "UserResponse", - { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - "xp": user.get("xp", 0), - "level": user.get("level", 1), - "badges": user.get("badges", []), - }, - ) - - -# ── 2FA / TOTP Endpoints ── - - -@router.post("/2fa/setup") -async def twofa_setup(request: Request) -> TwoFASetupResponse: - """Begin 2FA setup - generate secret + QR code. Returns plaintext secret + backup codes (shown once).""" - if not PYOTP_AVAILABLE: - raise HTTPException(status_code=503, detail="2FA service unavailable") - - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - - # Check if already enabled - if user.get("totp_secret"): - raise HTTPException( - status_code=400, detail="2FA already enabled. Disable first to reconfigure." - ) - - secret = _generate_totp_secret() - uri = _get_totp_uri(secret, user["email"]) - qr_b64 = _generate_qr_base64(uri) - backup_codes = _generate_backup_codes(8) - - # Store pending secret (not enabled until verified) - r = get_redis() - assert r is not None - pending = { - "secret": secret, - "backup_codes": [_hash_backup_code(c) for c in backup_codes], - "created_at": datetime.utcnow().isoformat(), - } - r.setex(f"rmi:2fa_pending:{user['id']}", timedelta(minutes=10), json.dumps(pending)) - - return cast("TwoFASetupResponse", { - "secret": secret, - "qr_code": qr_b64, - "uri": uri, - "backup_codes": backup_codes, # plaintext - shown once - }) - - -@router.post("/2fa/enable") -async def twofa_enable(req: TwoFAEnableRequest, request: Request) -> dict[str, Any]: - """Enable 2FA - verify the first TOTP code, then persist.""" - if not PYOTP_AVAILABLE: - raise HTTPException(status_code=503, detail="2FA service unavailable") - - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - - r = get_redis() - assert r is not None - pending_raw = r.get(f"rmi:2fa_pending:{user['id']}") - if not pending_raw: - raise HTTPException(status_code=400, detail="No pending 2FA setup. Call /2fa/setup first.") - - pending = json.loads(pending_raw) - secret = pending["secret"] - - # Verify the code - if not _verify_totp(secret, req.code): - raise HTTPException( - status_code=400, detail="Invalid TOTP code. Check your authenticator app time sync." - ) - - # Enable 2FA on user record - user["totp_secret"] = secret - user["totp_enabled"] = True - user["totp_created_at"] = pending["created_at"] - user["totp_backup_codes"] = pending["backup_codes"] # already hashed - user["totp_last_used"] = None - _save_user(user) - - # Clean up pending - r.delete(f"rmi:2fa_pending:{user['id']}") - - # Audit log - logger.info(f"[2FA ENABLED] user={user['id']} email={user['email']}") - - return {"status": "ok", "message": "2FA enabled successfully", "method": "totp"} - - -@router.post("/2fa/disable") -async def twofa_disable(request: Request) -> dict[str, Any]: - """Disable 2FA - requires current password for security.""" - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - - if not user.get("totp_enabled"): - raise HTTPException(status_code=400, detail="2FA is not enabled") - - # Remove 2FA fields - for key in [ - "totp_secret", - "totp_enabled", - "totp_created_at", - "totp_backup_codes", - "totp_last_used", - ]: - user.pop(key, None) - - _save_user(user) - logger.info(f"[2FA DISABLED] user={user['id']} email={user['email']}") - - return cast("dict[str, Any]", {"status": "ok", "message": "2FA disabled successfully"}) - - -@router.get("/2fa/status", response_model=TwoFAStatusResponse) -async def twofa_status(request: Request) -> TwoFAStatusResponse: - """Check 2FA status for current user.""" - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - - enabled = user.get("totp_enabled", False) - return cast("TwoFAStatusResponse", { - "enabled": enabled, - "method": "totp" if enabled else "none", - "created_at": user.get("totp_created_at"), - "last_used": user.get("totp_last_used"), - }) - - -@router.post("/2fa/verify") -async def twofa_verify(req: TwoFAVerifyRequest, request: Request) -> dict[str, Any]: - """Verify a TOTP code (for re-auth during sensitive operations).""" - if not PYOTP_AVAILABLE: - raise HTTPException(status_code=503, detail="2FA service unavailable") - - user = await get_current_user(request) - if not user: - raise HTTPException(status_code=401, detail="Authentication required") - - if not user.get("totp_enabled") or not user.get("totp_secret"): - raise HTTPException(status_code=400, detail="2FA not enabled") - - if not _verify_totp(user["totp_secret"], req.code): - raise HTTPException(status_code=400, detail="Invalid TOTP code") - - # Update last used - user["totp_last_used"] = datetime.utcnow().isoformat() - _save_user(user) - - return cast("dict[str, Any]", {"status": "ok", "verified": True}) - - -@router.post("/2fa/login") -async def twofa_login(req: TwoFALoginRequest) -> WalletAuthResponse: - """Login with email + password + 2FA code. Returns full JWT on success.""" - if not PYOTP_AVAILABLE: - raise HTTPException(status_code=503, detail="2FA service unavailable") - - user = _get_user_by_email(req.email) - if not user or not user.get("password_hash"): - raise HTTPException(status_code=401, detail="Invalid credentials") - - if not verify_password(req.password, user["password_hash"]): - raise HTTPException(status_code=401, detail="Invalid credentials") - - # Check if 2FA is enabled - if not user.get("totp_enabled") or not user.get("totp_secret"): - raise HTTPException( - status_code=400, detail="2FA not enabled for this account. Use /login instead." - ) - - code = req.code.strip().replace("-", "") - - # Try TOTP first - if _verify_totp(user["totp_secret"], code): - pass # valid TOTP - else: - # Try backup codes - backup_codes = user.get("totp_backup_codes", []) - matched = False - for i, hashed in enumerate(backup_codes): - if _verify_backup_code(req.code, hashed): - matched = True - # Remove used backup code - backup_codes.pop(i) - user["totp_backup_codes"] = backup_codes - break - if not matched: - raise HTTPException(status_code=401, detail="Invalid 2FA code or backup code") - - # Update last used - user["totp_last_used"] = datetime.utcnow().isoformat() - _save_user(user) - - token = _create_jwt( - user["id"], - user["email"], - user.get("tier", "FREE"), - user.get("role", "USER"), - user.get("wallet_address"), - ) - - logger.info(f"[2FA LOGIN] user={user['id']} email={user['email']}") - - return cast( - "WalletAuthResponse", - { - "access_token": token, - "refresh_token": token, - "user": { - "id": user["id"], - "email": user["email"], - "display_name": user.get("display_name", user["email"]), - "wallet_address": user.get("wallet_address"), - "wallet_chain": user.get("wallet_chain"), - "tier": user.get("tier", "FREE"), - "role": user.get("role", "USER"), - "created_at": user.get("created_at"), - }, - }, - ) +__all__ = [ + "EmailLoginRequest", + "EmailRegisterRequest", + "GoogleAuthResponse", + "JWT_ALGORITHM", + "JWT_EXPIRY_DAYS", + "JWT_SECRET", + "NonceResponse", + "PYOTP_AVAILABLE", + "PERMISSIONS", + "TOTP_DIGITS", + "TOTP_INTERVAL", + "TOTP_ISSUER", + "TOTP_WINDOW", + "TelegramAuthRequest", + "TwoFAEnableRequest", + "TwoFALoginRequest", + "TwoFASetupResponse", + "TwoFAStatusResponse", + "TwoFAVerifyRequest", + "UserResponse", + "WalletAuthResponse", + "WalletNonceRequest", + "WalletVerifyRequest", + "_build_totp", + "_create_jwt", + "_delete_user", + "_derive_user_id", + "_generate_backup_codes", + "_generate_qr_base64", + "_generate_totp_secret", + "_get_totp_uri", + "_get_user", + "_get_user_by_email", + "_hash_backup_code", + "_is_valid_email", + "_save_user", + "_verify_backup_code", + "_verify_jwt", + "_verify_totp", + "generate_nonce", + "get_current_user", + "hash_password", + "oauth_router", + "require_auth", + "require_public_profile", + "router", + "verify_password", +] \ No newline at end of file diff --git a/app/domains/auth/router.py b/app/domains/auth/router.py new file mode 100644 index 0000000..c11a476 --- /dev/null +++ b/app/domains/auth/router.py @@ -0,0 +1,509 @@ +""" +Auth Router - Complete authentication system (email, wallet, OAuth, Telegram) + +This is the FastAPI router for the auth domain. It is mounted by +``app/mount.py`` via the re-export in ``app.domains.auth.router``. + +Phase 2.1: extracted from ``app/domains/auth/__init__.py`` so that the +package marker is a clean re-export shim and mypy does not see the +router file as both module and package. +""" + +import json +import logging +from datetime import datetime, timedelta +from typing import Any, cast + +from fastapi import APIRouter, HTTPException, Request + +from app.core.redis import get_redis +from app.domains.auth.deps import ( + get_current_user, +) +from app.domains.auth.jwt import ( + JWT_ALGORITHM, + JWT_SECRET, + _create_jwt, + _derive_user_id, + _verify_jwt, + generate_nonce, +) +from app.domains.auth.oauth import router as oauth_router +from app.domains.auth.passwords import hash_password, verify_password +from app.domains.auth.schemas import ( + EmailLoginRequest, + EmailRegisterRequest, + NonceResponse, + TwoFAEnableRequest, + TwoFALoginRequest, + TwoFASetupResponse, + TwoFAStatusResponse, + TwoFAVerifyRequest, + UserResponse, + WalletAuthResponse, + WalletNonceRequest, + WalletVerifyRequest, +) +from app.domains.auth.store import ( + _get_user, + _get_user_by_email, + _is_valid_email, + _save_user, +) +from app.domains.auth.totp import ( + PYOTP_AVAILABLE, + _generate_backup_codes, + _generate_qr_base64, + _generate_totp_secret, + _get_totp_uri, + _hash_backup_code, + _verify_backup_code, + _verify_totp, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["auth"]) + +# Include OAuth/Telegram routes from dedicated submodule +router.include_router(oauth_router, tags=["auth"]) + +# ── Config ── + + +# ── Storage (Redis-backed user store) ── +# ── Email Auth ── +@router.post("/register", response_model=WalletAuthResponse) +def register_email(req: EmailRegisterRequest) -> WalletAuthResponse: + """Register a new user with email/password.""" + if not _is_valid_email(req.email): + raise HTTPException(status_code=400, detail="Invalid email format") + + if len(req.password) < 8: + raise HTTPException(status_code=400, detail="Password must be at least 8 characters") + + # Check if user exists + existing = _get_user_by_email(req.email) + if existing: + raise HTTPException(status_code=400, detail="Email already registered") + + user_id = _derive_user_id(req.email) + hashed = hash_password(req.password) + + user: dict[str, Any] = { + "id": user_id, + "email": req.email, + "display_name": req.display_name, + "password_hash": hashed, + "wallet_address": req.wallet_address, + "wallet_chain": req.wallet_chain, + "tier": "FREE", + "role": "USER", + "created_at": datetime.utcnow().isoformat(), + "xp": 0, + "level": 1, + "badges": [], + "scans_remaining": 5, + "scans_used": 0, + } + + # Save user + email index + r = get_redis() + assert r is not None + r.hset("rmi:users", user_id, json.dumps(user)) + r.hset("rmi:users:email", req.email.lower(), user_id) + + token = _create_jwt(user_id, req.email, "FREE", "USER", req.wallet_address) + + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user_id, + "email": req.email, + "display_name": req.display_name, + "wallet_address": req.wallet_address, + "wallet_chain": req.wallet_chain, + "tier": "FREE", + "role": "USER", + "created_at": user["created_at"], + }, + }, + ) + + +@router.post("/login", response_model=WalletAuthResponse) +def login_email(req: EmailLoginRequest) -> WalletAuthResponse: + """Login with email/password. If 2FA enabled, returns partial token requiring /2fa/login.""" + user = _get_user_by_email(req.email) + if not user or not user.get("password_hash"): + raise HTTPException(status_code=401, detail="Invalid credentials") + + if not verify_password(req.password, user["password_hash"]): + raise HTTPException(status_code=401, detail="Invalid credentials") + + # If 2FA enabled, return a pre-auth token that requires TOTP verification + if user.get("totp_enabled") and user.get("totp_secret"): + # Generate a short-lived pre-auth token (5 min) + pre_token = _create_jwt( + user["id"], + user["email"], + user.get("tier", "FREE"), + user.get("role", "USER"), + user.get("wallet_address"), + ) + # Override expiry to 5 minutes + import jose.jwt as jose_jwt + + payload = jose_jwt.decode(pre_token, JWT_SECRET, algorithms=[JWT_ALGORITHM]) + payload["exp"] = datetime.utcnow() + timedelta(minutes=5) + payload["2fa_required"] = True + payload["pre_auth"] = True + pre_token = jose_jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) + + return cast( + "WalletAuthResponse", + { + "access_token": pre_token, + "refresh_token": pre_token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + "_2fa_required": True, + }, + }, + ) + + token = _create_jwt( + user["id"], + user["email"], + user.get("tier", "FREE"), + user.get("role", "USER"), + user.get("wallet_address"), + ) + + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, + }, + ) + + +# ── Wallet Auth ── +@router.post("/wallet/nonce", response_model=NonceResponse) +async def wallet_nonce(req: WalletNonceRequest) -> NonceResponse: + """Get a nonce for wallet signature.""" + nonce = generate_nonce() + timestamp = datetime.utcnow().isoformat() + message = f"RugMunch Intelligence wants you to sign in with your {req.chain.title()} account.\n\nWallet: {req.address}\nNonce: {nonce}\nTimestamp: {timestamp}" + return cast("NonceResponse", {"nonce": nonce, "timestamp": timestamp, "message": message}) + + +@router.post("/wallet/verify", response_model=WalletAuthResponse) +async def wallet_verify(req: WalletVerifyRequest) -> WalletAuthResponse: + """Verify wallet signature and create session.""" + from app.domains.auth.wallet import get_or_create_wallet_user, verify_wallet_signature + + if not verify_wallet_signature(req.message, req.signature, req.address): + raise HTTPException(status_code=401, detail="Invalid signature") + + user_data = await get_or_create_wallet_user(req.address) + + return cast( + "WalletAuthResponse", + { + "access_token": user_data["access_token"], + "refresh_token": user_data["refresh_token"], + "user": { + "id": user_data["id"], + "email": user_data["email"], + "display_name": user_data.get("display_name", f"Agent {req.address[2:8].upper()}"), + "wallet_address": req.address, + "wallet_chain": req.chain, + "tier": user_data["tier"], + "role": user_data["role"], + "created_at": user_data["created_at"], + }, + }, + ) + + +# ── User Profile ── +@router.get("/user/me", response_model=UserResponse) +async def user_me(request: Request) -> UserResponse: + """Get current authenticated user profile.""" + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Missing auth token") + + token = auth_header[7:] + payload = _verify_jwt(token) + if not payload: + raise HTTPException(status_code=401, detail="Invalid token") + + user = _get_user(payload["id"]) + if not user: + raise HTTPException(status_code=404, detail="User not found") + + return cast( + "UserResponse", + { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + "xp": user.get("xp", 0), + "level": user.get("level", 1), + "badges": user.get("badges", []), + }, + ) + + +# ── 2FA / TOTP Endpoints ── + + +@router.post("/2fa/setup") +async def twofa_setup(request: Request) -> TwoFASetupResponse: + """Begin 2FA setup - generate secret + QR code. Returns plaintext secret + backup codes (shown once).""" + if not PYOTP_AVAILABLE: + raise HTTPException(status_code=503, detail="2FA service unavailable") + + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + + # Check if already enabled + if user.get("totp_secret"): + raise HTTPException( + status_code=400, detail="2FA already enabled. Disable first to reconfigure." + ) + + secret = _generate_totp_secret() + uri = _get_totp_uri(secret, user["email"]) + qr_b64 = _generate_qr_base64(uri) + backup_codes = _generate_backup_codes(8) + + # Store pending secret (not enabled until verified) + r = get_redis() + assert r is not None + pending = { + "secret": secret, + "backup_codes": [_hash_backup_code(c) for c in backup_codes], + "created_at": datetime.utcnow().isoformat(), + } + r.setex(f"rmi:2fa_pending:{user['id']}", timedelta(minutes=10), json.dumps(pending)) + + return cast( + "TwoFASetupResponse", + { + "secret": secret, + "qr_code": qr_b64, + "uri": uri, + "backup_codes": backup_codes, # plaintext - shown once + }, + ) + + +@router.post("/2fa/enable") +async def twofa_enable(req: TwoFAEnableRequest, request: Request) -> dict[str, Any]: + """Enable 2FA - verify the first TOTP code, then persist.""" + if not PYOTP_AVAILABLE: + raise HTTPException(status_code=503, detail="2FA service unavailable") + + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + + r = get_redis() + assert r is not None + pending_raw = r.get(f"rmi:2fa_pending:{user['id']}") + if not pending_raw: + raise HTTPException(status_code=400, detail="No pending 2FA setup. Call /2fa/setup first.") + + pending = json.loads(pending_raw) + secret = pending["secret"] + + # Verify the code + if not _verify_totp(secret, req.code): + raise HTTPException( + status_code=400, detail="Invalid TOTP code. Check your authenticator app time sync." + ) + + # Enable 2FA on user record + user["totp_secret"] = secret + user["totp_enabled"] = True + user["totp_created_at"] = pending["created_at"] + user["totp_backup_codes"] = pending["backup_codes"] # already hashed + user["totp_last_used"] = None + _save_user(user) + + # Clean up pending + r.delete(f"rmi:2fa_pending:{user['id']}") + + # Audit log + logger.info(f"[2FA ENABLED] user={user['id']} email={user['email']}") + + return {"status": "ok", "message": "2FA enabled successfully", "method": "totp"} + + +@router.post("/2fa/disable") +async def twofa_disable(request: Request) -> dict[str, Any]: + """Disable 2FA - requires current password for security.""" + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + + if not user.get("totp_enabled"): + raise HTTPException(status_code=400, detail="2FA is not enabled") + + # Remove 2FA fields + for key in [ + "totp_secret", + "totp_enabled", + "totp_created_at", + "totp_backup_codes", + "totp_last_used", + ]: + user.pop(key, None) + + _save_user(user) + logger.info(f"[2FA DISABLED] user={user['id']} email={user['email']}") + + return cast("dict[str, Any]", {"status": "ok", "message": "2FA disabled successfully"}) + + +@router.get("/2fa/status", response_model=TwoFAStatusResponse) +async def twofa_status(request: Request) -> TwoFAStatusResponse: + """Check 2FA status for current user.""" + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + + enabled = user.get("totp_enabled", False) + return cast( + "TwoFAStatusResponse", + { + "enabled": enabled, + "method": "totp" if enabled else "none", + "created_at": user.get("totp_created_at"), + "last_used": user.get("totp_last_used"), + }, + ) + + +@router.post("/2fa/verify") +async def twofa_verify(req: TwoFAVerifyRequest, request: Request) -> dict[str, Any]: + """Verify a TOTP code (for re-auth during sensitive operations).""" + if not PYOTP_AVAILABLE: + raise HTTPException(status_code=503, detail="2FA service unavailable") + + user = await get_current_user(request) + if not user: + raise HTTPException(status_code=401, detail="Authentication required") + + if not user.get("totp_enabled") or not user.get("totp_secret"): + raise HTTPException(status_code=400, detail="2FA not enabled") + + if not _verify_totp(user["totp_secret"], req.code): + raise HTTPException(status_code=400, detail="Invalid TOTP code") + + # Update last used + user["totp_last_used"] = datetime.utcnow().isoformat() + _save_user(user) + + return cast("dict[str, Any]", {"status": "ok", "verified": True}) + + +@router.post("/2fa/login") +async def twofa_login(req: TwoFALoginRequest) -> WalletAuthResponse: + """Login with email + password + 2FA code. Returns full JWT on success.""" + if not PYOTP_AVAILABLE: + raise HTTPException(status_code=503, detail="2FA service unavailable") + + user = _get_user_by_email(req.email) + if not user or not user.get("password_hash"): + raise HTTPException(status_code=401, detail="Invalid credentials") + + if not verify_password(req.password, user["password_hash"]): + raise HTTPException(status_code=401, detail="Invalid credentials") + + # Check if 2FA is enabled + if not user.get("totp_enabled") or not user.get("totp_secret"): + raise HTTPException( + status_code=400, detail="2FA not enabled for this account. Use /login instead." + ) + + code = req.code.strip().replace("-", "") + + # Try TOTP first + if _verify_totp(user["totp_secret"], code): + pass # valid TOTP + else: + # Try backup codes + backup_codes = user.get("totp_backup_codes", []) + matched = False + for i, hashed in enumerate(backup_codes): + if _verify_backup_code(req.code, hashed): + matched = True + # Remove used backup code + backup_codes.pop(i) + user["totp_backup_codes"] = backup_codes + break + if not matched: + raise HTTPException(status_code=401, detail="Invalid 2FA code or backup code") + + # Update last used + user["totp_last_used"] = datetime.utcnow().isoformat() + _save_user(user) + + token = _create_jwt( + user["id"], + user["email"], + user.get("tier", "FREE"), + user.get("role", "USER"), + user.get("wallet_address"), + ) + + logger.info(f"[2FA LOGIN] user={user['id']} email={user['email']}") + + return cast( + "WalletAuthResponse", + { + "access_token": token, + "refresh_token": token, + "user": { + "id": user["id"], + "email": user["email"], + "display_name": user.get("display_name", user["email"]), + "wallet_address": user.get("wallet_address"), + "wallet_chain": user.get("wallet_chain"), + "tier": user.get("tier", "FREE"), + "role": user.get("role", "USER"), + "created_at": user.get("created_at"), + }, + }, + ) \ No newline at end of file diff --git a/app/domains/mcp/__init__.py b/app/domains/mcp/__init__.py index 458132e..9372532 100644 --- a/app/domains/mcp/__init__.py +++ b/app/domains/mcp/__init__.py @@ -4,7 +4,7 @@ Phase 4 domain consolidation: app.mcp -> app.domains.mcp. """ from __future__ import annotations -from app.domains.mcp.registry import TOOL_CATEGORIES, get_tool_by_name +from app.domains.mcp.registry import TOOL_CATEGORIES, resolve_tool from app.domains.mcp.server import ( MCP_PROTOCOL_VERSION, MCP_SERVER_VERSION, @@ -28,6 +28,6 @@ __all__ = [ "TOOL_VERSIONS", "X402ToolManager", "call_tool", - "get_tool_by_name", "handle_mcp_call", + "resolve_tool", ] diff --git a/app/domains/telegram/rugmunchbot/bot.py b/app/domains/telegram/rugmunchbot/bot.py index 41c26a2..f7f3cc0 100644 --- a/app/domains/telegram/rugmunchbot/bot.py +++ b/app/domains/telegram/rugmunchbot/bot.py @@ -1640,20 +1640,39 @@ async def handle_message(update: Update, ctx: ContextTypes.DEFAULT_TYPE): await update.message.chat.send_action("typing") - # Try RMI RAG backend + # Try RAG search + LLM answer try: async with httpx.AsyncClient(timeout=30) as client: r = await client.post( - f"{BACKEND_URL}/api/v1/rag/query", - json={"query": text, "user_id": user.id, "context": "telegram_bot"}, + f"{BACKEND_URL}/api/v1/rag/v2/search", + json={"query": text, "top_k": 5}, ) + rag_context = "" if r.status_code == 200: - data = r.json() - answer = data.get("answer", "") - if answer: - db.increment_usage(user.id, ai_msg=True) - await update.message.reply_text(answer, parse_mode=ParseMode.HTML, reply_markup=back_kb()) - return + hits = r.json().get("hits", []) + rag_context = "\n\n".join(h.get("content", "")[:500] for h in hits[:3]) + + from app.ai_router import chat_completion + + messages = [ + { + "role": "system", + "content": ( + "You are RugMunch Intelligence, a crypto scam detection assistant. " + "Answer user questions about tokens, wallets, and scams concisely. " + f"Context from knowledge base:\n{rag_context}" if rag_context else + "You are RugMunch Intelligence, a crypto scam detection assistant. " + "Answer user questions about tokens, wallets, and scams concisely." + ), + }, + {"role": "user", "content": text}, + ] + result = await chat_completion(messages=messages, tier="T4", max_tokens=400, timeout=25.0) + answer = result.get("content", "") + if answer: + db.increment_usage(user.id, ai_msg=True) + await update.message.reply_text(answer, parse_mode=ParseMode.HTML, reply_markup=back_kb()) + return except Exception: pass diff --git a/app/mcp/__init__.py b/app/mcp/__init__.py index 593e3b3..41edf5f 100644 --- a/app/mcp/__init__.py +++ b/app/mcp/__init__.py @@ -16,21 +16,21 @@ from app.domains.mcp import ( TOOLS, X402ToolManager, call_tool, - get_tool_by_name, handle_mcp_call, + resolve_tool, ) __all__ = [ "MCP_PROTOCOL_VERSION", "MCP_SERVER_VERSION", - "TOOLS", "TOOL_CATALOG", "TOOL_CATEGORIES", "TOOL_DEPRECATED", "TOOL_SUCCESSORS", "TOOL_VERSIONS", + "TOOLS", "X402ToolManager", "call_tool", - "get_tool_by_name", "handle_mcp_call", + "resolve_tool", ] diff --git a/app/mount.py b/app/mount.py index 8f93d55..6e88bb1 100644 --- a/app/mount.py +++ b/app/mount.py @@ -42,6 +42,9 @@ ROUTER_MODULES: Final[list[str]] = [ "app.domains.news.admin_router", # /api/v1/news/_admin/* "app.domains.reports", # /api/v1/reports/* "app.domains.x402", # /api/v1/x402/* + "app.domains.databus.router", # /api/v1/databus/* (DataBus HTTP API) + "app.domains.admin.router", # /api/v1/admin/backend/* (admin login + RBAC) + "app.domains.auth.router", # /register, /login, /2fa/*, /wallet/*, /user/me, OAuth, Telegram # x402 MCP + Discovery + Docs "app.routers.x402_mcp_handler", # /mcp/x402, /.well-known/x402 "app.routers.x402_docs", # /x402/docs, /x402/sandbox/* diff --git a/app/routers/chat.py b/app/routers/chat.py index 91c0f8e..00c1b8a 100755 --- a/app/routers/chat.py +++ b/app/routers/chat.py @@ -40,7 +40,7 @@ ai_guard = AIGuard() # ── Auth ── # ── AI Router ── -from app.ai_router import router as ai_router # noqa: E402 +from app import ai_router # noqa: E402 from app.domains.auth import get_current_user, require_auth # noqa: E402 # ── DB path ── diff --git a/app/routers/wallet_clustering_router.py b/app/routers/wallet_clustering_router.py index 2bfbf08..4cfbb0e 100644 --- a/app/routers/wallet_clustering_router.py +++ b/app/routers/wallet_clustering_router.py @@ -151,7 +151,7 @@ async def get_bubble_map_data(req: BubblemapRequest): # AI Deep Dive: If requested and cluster is large/suspicious, trigger AI analysis if req.ai_deep_dive and data.get("nodes") and len(data["nodes"]) >= 5: try: - from app.ai_router import router as ai_router + from app import ai_router # Extract top suspicious wallets for AI context suspicious_wallets = [ @@ -178,7 +178,7 @@ async def get_bubble_map_data(req: BubblemapRequest): {"role": "user", "content": prompt}, ] - result = await ai_router.chat_completion( # type: ignore + result = await ai_router.chat_completion( messages=messages, tier="T2", temperature=0.2, max_tokens=400, timeout=15.0 ) @@ -217,7 +217,7 @@ async def get_ai_forensic_breakdown(req: BubblemapRequest): # If AI deep dive is explicitly requested, enrich with LLM analysis if req.ai_deep_dive and breakdown.get("total_wallets_analyzed", 0) > 0: try: - from app.ai_router import router as ai_router + from app import ai_router prompt = f"""You are an elite blockchain forensics analyst. Analyze this wallet cluster. @@ -246,7 +246,7 @@ Be direct and avoid fluff.""" {"role": "user", "content": prompt}, ] - result = await ai_router.chat_completion( # type: ignore + result = await ai_router.chat_completion( messages=messages, tier="T2", temperature=0.2, max_tokens=600, timeout=20.0 ) diff --git a/app/unified_wallet_scanner.py b/app/unified_wallet_scanner.py index e3d83eb..1787354 100644 --- a/app/unified_wallet_scanner.py +++ b/app/unified_wallet_scanner.py @@ -184,21 +184,26 @@ class UnifiedWalletScanner: # ═══════════════════════════════════════════════════════════════ async def _rag_enrich(self, r): try: - req = Request( - f"{BACKEND}/api/v1/rag/search?q={r.address} scam rug hack&limit=5", - headers={"X-RMI-Key": "rmi-internal-2026"}, - ) - resp = urlopen(req, timeout=8) - data = json.loads(resp.read()) - r.rag_matches = data.get("results", []) - if r.rag_matches: - r.modules_run.append("rag_entity_resolution") - r.enrichment_sources.append("rag:20K_docs") - for match in r.rag_matches[:3]: - content = match.get("content", "")[:80] - if r.address.lower() in content.lower(): - r.risk_flags.append("RAG_SCAMMER_MATCH") - r.risk_score += 30 + import httpx + + async with httpx.AsyncClient(timeout=8) as client: + resp = await client.post( + f"{BACKEND}/api/v1/rag/v2/search", + json={"query": f"{r.address} scam rug hack", "top_k": 5}, + headers={"X-RMI-Key": "rmi-internal-2026"}, + ) + if resp.status_code != 200: + return + data = resp.json() + r.rag_matches = data.get("hits", []) + if r.rag_matches: + r.modules_run.append("rag_entity_resolution") + r.enrichment_sources.append("rag:20K_docs") + for match in r.rag_matches[:3]: + content = match.get("content", "")[:80] + if r.address.lower() in content.lower(): + r.risk_flags.append("RAG_SCAMMER_MATCH") + r.risk_score += 30 except Exception as e: logger.warning(f"RAG enrich failed: {e}") diff --git a/infra/alert_rules.yml b/infra/alert_rules.yml index 6f09f0e..12e768c 100644 --- a/infra/alert_rules.yml +++ b/infra/alert_rules.yml @@ -16,8 +16,8 @@ groups: - alert: HighErrorRate expr: | ( - sum(rate(rmi_requests_total{status=~"5.."}[5m])) - / sum(rate(rmi_requests_total[5m])) + sum(rate(rmi_http_requests_total{status=~"5.."}[5m])) + / sum(rate(rmi_http_requests_total[5m])) ) > 0.05 for: 5m labels: @@ -133,8 +133,8 @@ groups: - alert: SLOBurnRate2x expr: | ( - sum(rate(rmi_requests_total{status=~"5.."}[1h])) - / sum(rate(rmi_requests_total[1h])) + sum(rate(rmi_http_requests_total{status=~"5.."}[1h])) + / sum(rate(rmi_http_requests_total[1h])) ) > 0.02 for: 5m labels: @@ -145,8 +145,8 @@ groups: - alert: SLOBurnRate4x expr: | ( - sum(rate(rmi_requests_total{status=~"5.."}[1h])) - / sum(rate(rmi_requests_total[1h])) + sum(rate(rmi_http_requests_total{status=~"5.."}[1h])) + / sum(rate(rmi_http_requests_total[1h])) ) > 0.04 for: 2m labels: diff --git a/requirements.txt b/requirements.txt index 64164c7..4b487e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,8 @@ python-telegram-bot>=21.0 supabase>=2.0.0 python-jose[cryptography]>=3.3.0 python-multipart>=0.0.17 +pyotp>=2.9.0 +qrcode>=7.4.0 aiohttp>=3.11.0 feedparser>=6.0.11 websockets>=13.0 From bd412acb2b96a6d6853fae06507a2a9fdd00e77d Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Tue, 7 Jul 2026 17:48:53 +0700 Subject: [PATCH 12/12] refactor(domains): merge token/tokens + scanner/scanners, drop empty app/domain/ - Move app/domains/tokens/deployer.py -> app/domains/token/deployer.py. Delete app/domains/tokens/ (was a single-file shadow of the token domain). - Move app/domains/scanner/* into app/domains/scanners/core/. The 31 detector modules in app/domains/scanners/ + the core service are now a single coherent domain. - Update import paths in app/token_scanner.py and app/token_deployer.py. - Delete empty app/domain/ (singular, was a leftover scaffold). - Factory still loads 466 routes, mypy-gate clean. --- .../{scanner => scanners/core}/__init__.py | 2 +- .../{scanner => scanners/core}/market_data.py | 2 +- .../{scanner => scanners/core}/models.py | 0 .../{scanner => scanners/core}/modules.py | 0 .../{scanner => scanners/core}/service.py | 10 +++++----- app/domains/{tokens => token}/deployer.py | 0 app/domains/tokens/__init__.py | 19 ------------------- app/token_deployer.py | 12 ++++++------ app/token_scanner.py | 2 +- 9 files changed, 14 insertions(+), 33 deletions(-) rename app/domains/{scanner => scanners/core}/__init__.py (90%) rename app/domains/{scanner => scanners/core}/market_data.py (98%) rename app/domains/{scanner => scanners/core}/models.py (100%) rename app/domains/{scanner => scanners/core}/modules.py (100%) rename app/domains/{scanner => scanners/core}/service.py (97%) rename app/domains/{tokens => token}/deployer.py (100%) delete mode 100644 app/domains/tokens/__init__.py diff --git a/app/domains/scanner/__init__.py b/app/domains/scanners/core/__init__.py similarity index 90% rename from app/domains/scanner/__init__.py rename to app/domains/scanners/core/__init__.py index 64b7952..adb4b64 100644 --- a/app/domains/scanner/__init__.py +++ b/app/domains/scanners/core/__init__.py @@ -15,6 +15,6 @@ Architecture: - app/token_scanner.py Backward compatibility shim """ -from app.domains.scanner.service import ScanResult, quick_scan_text, scan_token +from app.domains.scanners.core.service import ScanResult, quick_scan_text, scan_token __all__ = ["ScanResult", "quick_scan_text", "scan_token"] diff --git a/app/domains/scanner/market_data.py b/app/domains/scanners/core/market_data.py similarity index 98% rename from app/domains/scanner/market_data.py rename to app/domains/scanners/core/market_data.py index 4fd7923..8b6bf4c 100644 --- a/app/domains/scanner/market_data.py +++ b/app/domains/scanners/core/market_data.py @@ -4,7 +4,7 @@ from typing import Any import httpx -from app.domains.scanner.models import SOLANA_RPC_ENDPOINTS +from app.domains.scanners.core.models import SOLANA_RPC_ENDPOINTS logger = __import__("app.core.logging", fromlist=["get_logger"]).get_logger(__name__) diff --git a/app/domains/scanner/models.py b/app/domains/scanners/core/models.py similarity index 100% rename from app/domains/scanner/models.py rename to app/domains/scanners/core/models.py diff --git a/app/domains/scanner/modules.py b/app/domains/scanners/core/modules.py similarity index 100% rename from app/domains/scanner/modules.py rename to app/domains/scanners/core/modules.py diff --git a/app/domains/scanner/service.py b/app/domains/scanners/core/service.py similarity index 97% rename from app/domains/scanner/service.py rename to app/domains/scanners/core/service.py index f149461..47396a4 100644 --- a/app/domains/scanner/service.py +++ b/app/domains/scanners/core/service.py @@ -4,7 +4,7 @@ import asyncio from typing import Any from app.core.logging import get_logger -from app.domains.scanner.modules import ( +from app.domains.scanners.core.modules import ( _get_deployer_info, _get_holder_data, ) @@ -12,9 +12,9 @@ from app.domains.scanner.modules import ( logger = get_logger(__name__) # Re-export ScanResult for backward compatibility -from app.domains.scanner.market_data import fetch_market_data # noqa: E402 -from app.domains.scanner.models import ScanResult # noqa: E402 -from app.domains.scanner.modules import ( # noqa: E402 +from app.domains.scanners.core.market_data import fetch_market_data # noqa: E402 +from app.domains.scanners.core.models import ScanResult # noqa: E402 +from app.domains.scanners.core.modules import ( # noqa: E402 _check_blockscout, _check_defi_scanner, _check_honeypot_is, @@ -297,4 +297,4 @@ async def quick_scan_text(token_address: str, chain: str = "solana") -> str: # Backward compatibility: re-export scan_token -from app.domains.scanner.service import scan_token # noqa: E402, F811 +from app.domains.scanners.core.service import scan_token # noqa: E402, F811 diff --git a/app/domains/tokens/deployer.py b/app/domains/token/deployer.py similarity index 100% rename from app/domains/tokens/deployer.py rename to app/domains/token/deployer.py diff --git a/app/domains/tokens/__init__.py b/app/domains/tokens/__init__.py deleted file mode 100644 index 3d1089f..0000000 --- a/app/domains/tokens/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Tokens subsystem. - -Phase 3B of AUDIT-2026-Q3.md. - -Re-exports the canonical public API of the token deployer. Implementation -lives in app.tokens.deployer (moved verbatim from app.token_deployer on -2026-07-07). -""" -from app.domains.tokens.deployer import ( # noqa: F401 - ChainDeployer, - DeployParams, - DeploymentStorage, - EVMDeployer, - SolanaDeployer, - TokenDeployment, - TokenDeployerFactory, - TronDeployer, - get_storage, -) diff --git a/app/token_deployer.py b/app/token_deployer.py index 75df8ff..781509f 100644 --- a/app/token_deployer.py +++ b/app/token_deployer.py @@ -1,19 +1,19 @@ -"""token_deployer.py - DEPRECATED shim. Use app.domains.tokens.deployer. +"""token_deployer.py - DEPRECATED shim. Use app.domains.token.deployer. -Phase 4 of AUDIT-2026-Q3.md moved this to app/domains/tokens/deployer/. +Phase 4 of AUDIT-2026-Q3.md moved this to app/domains/token/deployer/. This shim re-exports the public surface for legacy callers. """ -from app.domains.tokens.deployer import * # noqa: F401,F403 -from app.domains.tokens.deployer import ( # noqa: F401 +from app.domains.token.deployer import * # noqa: F403 +from app.domains.token.deployer import ( # noqa: F401 ChainDeployer, - DeployParams, DeploymentStorage, + DeployParams, EVMDeployer, SolanaDeployer, TokenDeployerFactory, TokenDeployment, TronDeployer, + _storage, get_storage, logger, - _storage, ) diff --git a/app/token_scanner.py b/app/token_scanner.py index d0644b4..3407b4d 100644 --- a/app/token_scanner.py +++ b/app/token_scanner.py @@ -6,6 +6,6 @@ better organization (per architecture standard: NO file > 500 lines). This file re-exports the main API from the new modules. """ -from app.domains.scanner import ScanResult, quick_scan_text, scan_token +from app.domains.scanners.core import ScanResult, quick_scan_text, scan_token __all__ = ["ScanResult", "quick_scan_text", "scan_token"]