cryptorugmunch
628c1d2a10
Some checks failed
CI / build (push) Failing after 3s
PHASE 2.3 (AUDIT-2026-Q3.md):
Task 1 — Wire-in Wave 3 (1 router mounted, 2 deferred):
- app.routers.unified_scanner_router mounted at /api/v2/scanner/* (2 routes:
POST /api/v2/scanner/token/scan, POST /api/v2/scanner/wallet/scan).
Refactored prefix from /api/v2 -> /api/v2/scanner to avoid future conflicts
with the v1 /api/v1/scanner/ stub.
- app.routers.unified_wallet_scanner DEFERRED (no router APIRouter attribute;
library module consumed by unified_scanner_router via get_wallet_scanner()).
- app.routers.admin_extensions DEFERRED (DORMANT per audit; 25 routes at
/api/v1/admin/* would shadow /api/v1/admin/alerts_webhook).
Task 2 — Archive 136 dead-code files to app/_archive/legacy_2026_07/:
- 73 routers in app/routers/ (reach graph showed zero reach into mount.py).
- 63 flat app/*.py (domain modules never imported by live code).
- 1 file RESTORED post-archive: app/routers/x402_bridge_health.py (caught by
tests/unit/test_bridge_health.py which directly imports it; reach graph
considered tests/ only as transitive reach — to be patched in next cycle).
Forced-LIVE (NOT archived per user directive):
- app/ai_pipeline_v3.py (3 importers in audit window, importers themselves DEAD)
- app/splade_bm25.py (LIVE via app.rag_service)
- app/wallet_manager_v2.py (LIVE via x402_enforcement, x402_tools, sweep_all, sweep_now)
- app/crypto_embeddings.py (NOT in audit ARCHIVE list; heavy import graph)
Verification (forward-import closure from mount.py + main.py + factory.py + lifespan.py):
- imports = 348 app.* modules
- reached = 194 files reachable from roots
- archive set = audit_dead (186) - reached - forced_live (4) - test_live (1) = 136
- Net delta: 136 files moved, 44,932 LOC reduction, 293->295 active routes (+2 from Wave 3)
pyproject.toml updates:
- setuptools.packages.find: added exclude for app._archive*
- ruff.extend-exclude: added "app/_archive/"
- mypy.exclude: added "app/_archive/"
Smoke test: pytest tests/ — 817 passed, 3 pre-existing failures unchanged
(0 new failures; 0 routes lost; all 4 forced-LIVE files still importable).
Restoration: git mv app/_archive/legacy_2026_07/<name>.py <original-path>
and add the import to app/mount.py ROUTER_MODULES.
Refs: AUDIT-2026-Q3.md /home/dev/pry/rmi-final-deadcode-2026-07-06.md
413 lines
15 KiB
Python
413 lines
15 KiB
Python
"""
|
|
RAGAS Evaluation Pipeline - Weekly RAG quality assessment.
|
|
=============================================================
|
|
Evaluates RAG system against golden test set using RAGAS metrics:
|
|
- faithfulness: is the answer grounded in retrieved context?
|
|
- answer_relevancy: does the answer address the question?
|
|
- context_precision: are retrieved chunks relevant?
|
|
- context_recall: are all relevant chunks retrieved?
|
|
|
|
Golden test set: 20 known crypto scam/intel queries with expected answers.
|
|
Runs weekly. Alerts on regression > 10% from baseline.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
from datetime import UTC, datetime
|
|
|
|
import httpx
|
|
|
|
BACKEND = "http://localhost:8000"
|
|
RAG_SEARCH = f"{BACKEND}/api/v1/rag/search"
|
|
|
|
# ═══════════════════════════════════════════════════
|
|
# GOLDEN TEST SET - 20 queries with expected answers
|
|
# ═══════════════════════════════════════════════════
|
|
GOLDEN_TESTS = [
|
|
{
|
|
"query": "What are the most common DeFi hack techniques?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": [
|
|
"private key",
|
|
"flash loan",
|
|
"oracle manipulation",
|
|
"reentrancy",
|
|
"access control",
|
|
"supply chain",
|
|
],
|
|
"min_results": 3,
|
|
},
|
|
{
|
|
"query": "How much was stolen in the Bybit hack?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["bybit", "1.4", "billion", "1.46"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a honeypot scam in crypto?",
|
|
"collection": "scam_patterns",
|
|
"expected_terms": ["honeypot", "sell", "restriction", "cannot sell", "maxTx"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "What are the signs of a rug pull?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["liquidity", "remove", "lp", "supply", "concentration", "fresh wallet"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "How do crypto money launderers operate?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["chain hopping", "mixer", "peel chain", "cex", "cross chain"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "What did the Chainalysis 2025 crime report find?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["40.9", "billion", "illicit", "stablecoin", "63%", "stolen"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What are the top smart contract vulnerabilities?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["access control", "reentrancy", "oracle", "private key", "flash loan"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "How much did North Korean hackers steal in 2024?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["north korea", "1.34", "billion", "61%"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What happened with the Squid Game token?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["squid", "game", "honeypot", "couldn't sell", "3.38"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a bundle attack in crypto?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["bundle", "sniper", "mempool", "same block", "coordinated"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How does wash trading work in crypto?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["wash trading", "circular", "volume", "inflation", "same entity"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is the biggest DeFi hack ever?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["bybit", "ronin", "poly network", "billion"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "What are pig butchering scams?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["pig butchering", "romance", "investment", "scam"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How do pump and dump schemes work in crypto?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["pump", "dump", "insider", "accumulation", "fomo", "social"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What did TRM Labs report about crypto crime in 2025?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["158", "billion", "illicit", "A7A5", "russia", "sanctions"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What percentage of bug bounty programs find critical bugs?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["93.9%", "critical", "bug bounty", "immunefi"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is the SafeMoon scam?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["safemoon", "liquidity", "drain", "arrested", "fraud"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How are crypto scams using AI?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["AI", "deepfake", "personalized", "KYC", "bypass"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a flash loan attack?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["flash loan", "uncollateralized", "manipulate", "arbitrage"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What are the signs of a honeypot token contract?",
|
|
"collection": "scam_patterns",
|
|
"expected_terms": ["honeypot", "sell", "restriction", "maxTx", "trading", "owner"],
|
|
"min_results": 2,
|
|
},
|
|
# ── Expanded tests (30 new) ──
|
|
{
|
|
"query": "What is the OWASP Smart Contract Top 10?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["access control", "business logic", "oracle", "reentrancy", "proxy"],
|
|
"min_results": 3,
|
|
},
|
|
{
|
|
"query": "How was the Ronin Bridge hacked?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["ronin", "bridge", "validator", "private key"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What are common smart contract audit checklist items?",
|
|
"collection": "contract_audits",
|
|
"expected_terms": ["access control", "overflow", "reentrancy", "oracle", "validation"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "How does a flash loan attack work?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["flash loan", "uncollateralized", "single transaction", "arbitrage"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What happened in the OneCoin scam?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["onecoin", "ponzi", "4 billion", "bitcoin"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a private key compromise attack?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["private key", "compromise", "hot wallet", "phishing"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "How do cross-chain bridge hacks work?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["bridge", "cross chain", "validator", "message"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "What are the top smart contract vulnerabilities in 2025?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["access control", "reentrancy", "oracle", "overflow", "proxy"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "How did the Squid Game token scam work?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["squid", "game", "honeypot", "sell", "2,861"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a supply chain attack in crypto?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["supply chain", "ads", "power", "bybit", "compromise"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How much crypto was stolen in 2024?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["2.2", "billion", "stolen", "40.9"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is the GMX V1 vulnerability?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["gmx", "reentrancy", "glp", "arbitrum"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How does Tornado Cash work in laundering?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["tornado", "mixer", "launder", "privacy"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is an access control vulnerability?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["access control", "unauthorized", "privileged", "owner"],
|
|
"min_results": 2,
|
|
},
|
|
{
|
|
"query": "How did BitConnect scam investors?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["bitconnect", "ponzi", "40%", "2 billion"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What happened with the Poly Network hack?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["poly network", "610", "cross chain", "white hat"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How do North Korean hackers steal crypto?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["north korea", "lazarus", "1.34", "61%", "IT workers"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a proxy upgrade vulnerability?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["proxy", "upgrade", "implementation", "initialize"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How does the SENTINEL scanner detect scams?",
|
|
"collection": "known_scams",
|
|
"expected_terms": ["scam", "detect", "honeypot", "rug"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What are common DeFi money laundering patterns?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["peel chain", "mixer", "chain hop", "cex"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How did the Euler Finance hack happen?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["euler", "flash loan", "197", "donate"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What percentage of stolen crypto is from private key compromises?",
|
|
"collection": "crime_reports",
|
|
"expected_terms": ["43.8%", "private key", "stolen", "compromise"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How do oracle manipulation attacks work?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["oracle", "price", "manipulation", "flash loan", "twap"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is a reentrancy attack in Solidity?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["reentrancy", "callback", "withdraw", "state"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How did SafeMoon defraud investors?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["safemoon", "liquidity", "drain", "arrest"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What are the biggest DeFi hacks by amount lost?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["bybit", "ronin", "poly", "billion", "million"],
|
|
"min_results": 3,
|
|
},
|
|
{
|
|
"query": "How does a sniper bot attack tokens at launch?",
|
|
"collection": "transaction_patterns",
|
|
"expected_terms": ["sniper", "mempool", "same block", "gas"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What is business logic vulnerability in smart contracts?",
|
|
"collection": "vuln_patterns",
|
|
"expected_terms": ["business logic", "design", "lending", "amm", "reward"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "How did the Thodex exchange scam work?",
|
|
"collection": "rug_timeline",
|
|
"expected_terms": ["thodex", "exchange", "2 billion", "turkey"],
|
|
"min_results": 1,
|
|
},
|
|
{
|
|
"query": "What are the most exploited chains for DeFi hacks?",
|
|
"collection": "defi_hacks",
|
|
"expected_terms": ["ethereum", "bsc", "polygon", "arbitrum", "solana"],
|
|
"min_results": 2,
|
|
},
|
|
]
|
|
|
|
|
|
async def evaluate_single(test: dict) -> dict:
|
|
"""Run a single test query and score it."""
|
|
query = test["query"]
|
|
collection = test["collection"]
|
|
expected_terms = [t.lower() for t in test["expected_terms"]]
|
|
min_results = test["min_results"]
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=30) as c:
|
|
r = await c.get(
|
|
RAG_SEARCH,
|
|
params={
|
|
"q": query,
|
|
"collection": collection,
|
|
"limit": 10,
|
|
},
|
|
)
|
|
if r.status_code != 200:
|
|
return {"query": query, "error": f"HTTP {r.status_code}", "score": 0}
|
|
|
|
data = r.json()
|
|
results = data.get("results", [])
|
|
total = data.get("total", 0)
|
|
|
|
# Score: context_precision - how many expected terms appear in results?
|
|
all_text = " ".join([res.get("content", res.get("text", "")) for res in results]).lower()
|
|
terms_found = sum(1 for t in expected_terms if t in all_text)
|
|
precision = terms_found / len(expected_terms) if expected_terms else 0
|
|
|
|
# Score: context_recall - did we get enough results?
|
|
recall = min(total / min_results, 1.0) if min_results > 0 else 1.0
|
|
|
|
# Combined score
|
|
score = precision * 0.6 + recall * 0.4
|
|
|
|
return {
|
|
"query": query[:60],
|
|
"collection": collection,
|
|
"results_found": total,
|
|
"min_expected": min_results,
|
|
"terms_matched": f"{terms_found}/{len(expected_terms)}",
|
|
"precision": round(precision, 3),
|
|
"recall": round(recall, 3),
|
|
"score": round(score, 3),
|
|
}
|
|
except Exception as e:
|
|
return {"query": query[:60], "error": str(e)[:200], "score": 0}
|
|
|
|
|
|
async def run_evaluation() -> dict:
|
|
"""Run full evaluation against golden test set."""
|
|
results = []
|
|
for test in GOLDEN_TESTS:
|
|
result = await evaluate_single(test)
|
|
results.append(result)
|
|
|
|
scores = [r["score"] for r in results if "score" in r]
|
|
avg_score = sum(scores) / len(scores) if scores else 0
|
|
passing = sum(1 for s in scores if s >= 0.5)
|
|
failing = sum(1 for s in scores if s < 0.3)
|
|
|
|
return {
|
|
"test_count": len(GOLDEN_TESTS),
|
|
"tests_run": len(results),
|
|
"average_score": round(avg_score, 3),
|
|
"passing": passing,
|
|
"failing": failing,
|
|
"pass_rate": round(passing / len(scores), 3) if scores else 0,
|
|
"results": results,
|
|
"timestamp": datetime.now(UTC).isoformat(),
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
result = asyncio.run(run_evaluation())
|
|
print(json.dumps(result, indent=2))
|