""" RAGAS Evaluation Pipeline - Weekly RAG quality assessment. ============================================================= Evaluates RAG system against golden test set using RAGAS metrics: - faithfulness: is the answer grounded in retrieved context? - answer_relevancy: does the answer address the question? - context_precision: are retrieved chunks relevant? - context_recall: are all relevant chunks retrieved? Golden test set: 20 known crypto scam/intel queries with expected answers. Runs weekly. Alerts on regression > 10% from baseline. """ import asyncio import json from datetime import UTC, datetime import httpx BACKEND = "http://localhost:8000" RAG_SEARCH = f"{BACKEND}/api/v1/rag/search" # ═══════════════════════════════════════════════════ # GOLDEN TEST SET - 20 queries with expected answers # ═══════════════════════════════════════════════════ GOLDEN_TESTS = [ { "query": "What are the most common DeFi hack techniques?", "collection": "defi_hacks", "expected_terms": [ "private key", "flash loan", "oracle manipulation", "reentrancy", "access control", "supply chain", ], "min_results": 3, }, { "query": "How much was stolen in the Bybit hack?", "collection": "defi_hacks", "expected_terms": ["bybit", "1.4", "billion", "1.46"], "min_results": 1, }, { "query": "What is a honeypot scam in crypto?", "collection": "scam_patterns", "expected_terms": ["honeypot", "sell", "restriction", "cannot sell", "maxTx"], "min_results": 2, }, { "query": "What are the signs of a rug pull?", "collection": "rug_timeline", "expected_terms": ["liquidity", "remove", "lp", "supply", "concentration", "fresh wallet"], "min_results": 2, }, { "query": "How do crypto money launderers operate?", "collection": "transaction_patterns", "expected_terms": ["chain hopping", "mixer", "peel chain", "cex", "cross chain"], "min_results": 2, }, { "query": "What did the Chainalysis 2025 crime report find?", "collection": "crime_reports", "expected_terms": ["40.9", "billion", "illicit", "stablecoin", "63%", "stolen"], "min_results": 1, }, { "query": "What are the top smart contract vulnerabilities?", "collection": "vuln_patterns", "expected_terms": ["access control", "reentrancy", "oracle", "private key", "flash loan"], "min_results": 2, }, { "query": "How much did North Korean hackers steal in 2024?", "collection": "crime_reports", "expected_terms": ["north korea", "1.34", "billion", "61%"], "min_results": 1, }, { "query": "What happened with the Squid Game token?", "collection": "rug_timeline", "expected_terms": ["squid", "game", "honeypot", "couldn't sell", "3.38"], "min_results": 1, }, { "query": "What is a bundle attack in crypto?", "collection": "transaction_patterns", "expected_terms": ["bundle", "sniper", "mempool", "same block", "coordinated"], "min_results": 1, }, { "query": "How does wash trading work in crypto?", "collection": "transaction_patterns", "expected_terms": ["wash trading", "circular", "volume", "inflation", "same entity"], "min_results": 1, }, { "query": "What is the biggest DeFi hack ever?", "collection": "defi_hacks", "expected_terms": ["bybit", "ronin", "poly network", "billion"], "min_results": 2, }, { "query": "What are pig butchering scams?", "collection": "crime_reports", "expected_terms": ["pig butchering", "romance", "investment", "scam"], "min_results": 1, }, { "query": "How do pump and dump schemes work in crypto?", "collection": "transaction_patterns", "expected_terms": ["pump", "dump", "insider", "accumulation", "fomo", "social"], "min_results": 1, }, { "query": "What did TRM Labs report about crypto crime in 2025?", "collection": "crime_reports", "expected_terms": ["158", "billion", "illicit", "A7A5", "russia", "sanctions"], "min_results": 1, }, { "query": "What percentage of bug bounty programs find critical bugs?", "collection": "vuln_patterns", "expected_terms": ["93.9%", "critical", "bug bounty", "immunefi"], "min_results": 1, }, { "query": "What is the SafeMoon scam?", "collection": "rug_timeline", "expected_terms": ["safemoon", "liquidity", "drain", "arrested", "fraud"], "min_results": 1, }, { "query": "How are crypto scams using AI?", "collection": "crime_reports", "expected_terms": ["AI", "deepfake", "personalized", "KYC", "bypass"], "min_results": 1, }, { "query": "What is a flash loan attack?", "collection": "vuln_patterns", "expected_terms": ["flash loan", "uncollateralized", "manipulate", "arbitrage"], "min_results": 1, }, { "query": "What are the signs of a honeypot token contract?", "collection": "scam_patterns", "expected_terms": ["honeypot", "sell", "restriction", "maxTx", "trading", "owner"], "min_results": 2, }, # ── Expanded tests (30 new) ── { "query": "What is the OWASP Smart Contract Top 10?", "collection": "vuln_patterns", "expected_terms": ["access control", "business logic", "oracle", "reentrancy", "proxy"], "min_results": 3, }, { "query": "How was the Ronin Bridge hacked?", "collection": "defi_hacks", "expected_terms": ["ronin", "bridge", "validator", "private key"], "min_results": 1, }, { "query": "What are common smart contract audit checklist items?", "collection": "contract_audits", "expected_terms": ["access control", "overflow", "reentrancy", "oracle", "validation"], "min_results": 2, }, { "query": "How does a flash loan attack work?", "collection": "defi_hacks", "expected_terms": ["flash loan", "uncollateralized", "single transaction", "arbitrage"], "min_results": 1, }, { "query": "What happened in the OneCoin scam?", "collection": "rug_timeline", "expected_terms": ["onecoin", "ponzi", "4 billion", "bitcoin"], "min_results": 1, }, { "query": "What is a private key compromise attack?", "collection": "defi_hacks", "expected_terms": ["private key", "compromise", "hot wallet", "phishing"], "min_results": 2, }, { "query": "How do cross-chain bridge hacks work?", "collection": "defi_hacks", "expected_terms": ["bridge", "cross chain", "validator", "message"], "min_results": 2, }, { "query": "What are the top smart contract vulnerabilities in 2025?", "collection": "vuln_patterns", "expected_terms": ["access control", "reentrancy", "oracle", "overflow", "proxy"], "min_results": 2, }, { "query": "How did the Squid Game token scam work?", "collection": "rug_timeline", "expected_terms": ["squid", "game", "honeypot", "sell", "2,861"], "min_results": 1, }, { "query": "What is a supply chain attack in crypto?", "collection": "vuln_patterns", "expected_terms": ["supply chain", "ads", "power", "bybit", "compromise"], "min_results": 1, }, { "query": "How much crypto was stolen in 2024?", "collection": "crime_reports", "expected_terms": ["2.2", "billion", "stolen", "40.9"], "min_results": 1, }, { "query": "What is the GMX V1 vulnerability?", "collection": "defi_hacks", "expected_terms": ["gmx", "reentrancy", "glp", "arbitrum"], "min_results": 1, }, { "query": "How does Tornado Cash work in laundering?", "collection": "transaction_patterns", "expected_terms": ["tornado", "mixer", "launder", "privacy"], "min_results": 1, }, { "query": "What is an access control vulnerability?", "collection": "vuln_patterns", "expected_terms": ["access control", "unauthorized", "privileged", "owner"], "min_results": 2, }, { "query": "How did BitConnect scam investors?", "collection": "rug_timeline", "expected_terms": ["bitconnect", "ponzi", "40%", "2 billion"], "min_results": 1, }, { "query": "What happened with the Poly Network hack?", "collection": "defi_hacks", "expected_terms": ["poly network", "610", "cross chain", "white hat"], "min_results": 1, }, { "query": "How do North Korean hackers steal crypto?", "collection": "crime_reports", "expected_terms": ["north korea", "lazarus", "1.34", "61%", "IT workers"], "min_results": 1, }, { "query": "What is a proxy upgrade vulnerability?", "collection": "vuln_patterns", "expected_terms": ["proxy", "upgrade", "implementation", "initialize"], "min_results": 1, }, { "query": "How does the SENTINEL scanner detect scams?", "collection": "known_scams", "expected_terms": ["scam", "detect", "honeypot", "rug"], "min_results": 1, }, { "query": "What are common DeFi money laundering patterns?", "collection": "transaction_patterns", "expected_terms": ["peel chain", "mixer", "chain hop", "cex"], "min_results": 1, }, { "query": "How did the Euler Finance hack happen?", "collection": "defi_hacks", "expected_terms": ["euler", "flash loan", "197", "donate"], "min_results": 1, }, { "query": "What percentage of stolen crypto is from private key compromises?", "collection": "crime_reports", "expected_terms": ["43.8%", "private key", "stolen", "compromise"], "min_results": 1, }, { "query": "How do oracle manipulation attacks work?", "collection": "vuln_patterns", "expected_terms": ["oracle", "price", "manipulation", "flash loan", "twap"], "min_results": 1, }, { "query": "What is a reentrancy attack in Solidity?", "collection": "vuln_patterns", "expected_terms": ["reentrancy", "callback", "withdraw", "state"], "min_results": 1, }, { "query": "How did SafeMoon defraud investors?", "collection": "rug_timeline", "expected_terms": ["safemoon", "liquidity", "drain", "arrest"], "min_results": 1, }, { "query": "What are the biggest DeFi hacks by amount lost?", "collection": "defi_hacks", "expected_terms": ["bybit", "ronin", "poly", "billion", "million"], "min_results": 3, }, { "query": "How does a sniper bot attack tokens at launch?", "collection": "transaction_patterns", "expected_terms": ["sniper", "mempool", "same block", "gas"], "min_results": 1, }, { "query": "What is business logic vulnerability in smart contracts?", "collection": "vuln_patterns", "expected_terms": ["business logic", "design", "lending", "amm", "reward"], "min_results": 1, }, { "query": "How did the Thodex exchange scam work?", "collection": "rug_timeline", "expected_terms": ["thodex", "exchange", "2 billion", "turkey"], "min_results": 1, }, { "query": "What are the most exploited chains for DeFi hacks?", "collection": "defi_hacks", "expected_terms": ["ethereum", "bsc", "polygon", "arbitrum", "solana"], "min_results": 2, }, ] async def evaluate_single(test: dict) -> dict: """Run a single test query and score it.""" query = test["query"] collection = test["collection"] expected_terms = [t.lower() for t in test["expected_terms"]] min_results = test["min_results"] try: async with httpx.AsyncClient(timeout=30) as c: r = await c.get( RAG_SEARCH, params={ "q": query, "collection": collection, "limit": 10, }, ) if r.status_code != 200: return {"query": query, "error": f"HTTP {r.status_code}", "score": 0} data = r.json() results = data.get("results", []) total = data.get("total", 0) # Score: context_precision - how many expected terms appear in results? all_text = " ".join([res.get("content", res.get("text", "")) for res in results]).lower() terms_found = sum(1 for t in expected_terms if t in all_text) precision = terms_found / len(expected_terms) if expected_terms else 0 # Score: context_recall - did we get enough results? recall = min(total / min_results, 1.0) if min_results > 0 else 1.0 # Combined score score = precision * 0.6 + recall * 0.4 return { "query": query[:60], "collection": collection, "results_found": total, "min_expected": min_results, "terms_matched": f"{terms_found}/{len(expected_terms)}", "precision": round(precision, 3), "recall": round(recall, 3), "score": round(score, 3), } except Exception as e: return {"query": query[:60], "error": str(e)[:200], "score": 0} async def run_evaluation() -> dict: """Run full evaluation against golden test set.""" results = [] for test in GOLDEN_TESTS: result = await evaluate_single(test) results.append(result) scores = [r["score"] for r in results if "score" in r] avg_score = sum(scores) / len(scores) if scores else 0 passing = sum(1 for s in scores if s >= 0.5) failing = sum(1 for s in scores if s < 0.3) return { "test_count": len(GOLDEN_TESTS), "tests_run": len(results), "average_score": round(avg_score, 3), "passing": passing, "failing": failing, "pass_rate": round(passing / len(scores), 3) if scores else 0, "results": results, "timestamp": datetime.now(UTC).isoformat(), } if __name__ == "__main__": result = asyncio.run(run_evaluation()) print(json.dumps(result, indent=2))