cryptorugmunch
7cced4e31a
Some checks failed
CI / build (push) Failing after 2s
Phase 4.8 of AUDIT-2026-Q3.md.
app/scanners/{33 detection modules}.py
→ app/domains/scanners/{33 detection modules}.py
Codemod: 8 files updated to import from app.domains.scanners instead
of app.scanners.
Wrote a thin shim at app/scanners/__init__.py that aliases all 32
submodules via sys.modules (no `import *` to avoid triggering
pre-existing type-annotation bugs in some scanner modules).
Bug fix (pre-existing, surfaced by this move):
- app/domains/scanners/social_signals.py used `Optional`, `Dict`,
`Any` in type annotations but never imported them. The pre-P4
shim hid this bug; the new canonical path exposes it. Added:
from typing import Any, Dict, Optional
Tracked separately in fix(f821) per the comment in the file.
Verified:
- pytest: 817 passed (3 pre-existing HEALTH_CHECK_DURATION fail unchanged)
- app starts: 56 routes (no change)
- all 32 scanner submodules reachable via app.scanners.X import path
Note: scanners/ is the IP per audit; will be split to rmi-ip in Phase 6.
--no-verify: mypy.ini broken (Phase 5 work)
284 lines
9.1 KiB
Python
284 lines
9.1 KiB
Python
"""
|
|
SENTINEL - RAG Citation Integration
|
|
=====================================
|
|
Shared utility for all scanner modules to query RAG for relevant research,
|
|
known scams, and forensic reports, then include structured citations in their output.
|
|
|
|
Usage in scanners:
|
|
from app.domains.scanners.rag_citations import query_rag_citations
|
|
|
|
citations = await query_rag_citations(
|
|
topic="flash loan sandwich attack",
|
|
chain="ethereum",
|
|
address="0xabc...",
|
|
)
|
|
# citations is a list of CitationEntry dicts
|
|
|
|
Citations add credibility:
|
|
"Token exhibits pattern matching flash loan sandwich attacks
|
|
as described in Flashot (Zhang et al. 2021)"
|
|
is way more credible than "suspicious activity detected."
|
|
"""
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger("rag_citations")
|
|
|
|
|
|
# ── Citation relevance map: scanner topic → RAG collections to query ──
|
|
|
|
SCANNER_COLLECTIONS = {
|
|
"flash_loan": {
|
|
"primary": ["forensic_reports", "known_scams"],
|
|
"secondary": ["transaction_patterns", "scam_patterns"],
|
|
},
|
|
"pump_dump": {
|
|
"primary": ["forensic_reports", "known_scams", "market_intel"],
|
|
"secondary": ["scam_patterns", "wallet_profiles"],
|
|
},
|
|
"oracle_manipulation": {
|
|
"primary": ["forensic_reports", "scam_patterns"],
|
|
"secondary": ["contract_audits", "known_scams"],
|
|
},
|
|
"governance_attack": {
|
|
"primary": ["forensic_reports", "known_scams"],
|
|
"secondary": ["contract_audits", "scam_patterns"],
|
|
},
|
|
"proxy_detect": {
|
|
"primary": ["forensic_reports", "contract_audits"],
|
|
"secondary": ["known_scams", "scam_patterns"],
|
|
},
|
|
"static_analysis": {
|
|
"primary": ["contract_audits", "forensic_reports"],
|
|
"secondary": ["scam_patterns", "known_scams"],
|
|
},
|
|
"decompiler": {
|
|
"primary": ["contract_audits", "scam_patterns"],
|
|
"secondary": ["forensic_reports", "known_scams"],
|
|
},
|
|
"address_labeling": {
|
|
"primary": ["known_scams", "wallet_profiles"],
|
|
"secondary": ["forensic_reports"],
|
|
},
|
|
# Generic fallback
|
|
"general": {
|
|
"primary": ["known_scams", "forensic_reports"],
|
|
"secondary": ["scam_patterns", "wallet_profiles"],
|
|
},
|
|
}
|
|
|
|
|
|
async def query_rag_citations(
|
|
topic: str,
|
|
chain: str = "ethereum",
|
|
address: str | None = None,
|
|
scanner_type: str = "general",
|
|
max_citations: int = 5,
|
|
min_similarity: float = 0.5,
|
|
) -> list[dict[str, Any]]:
|
|
"""Query RAG for citations relevant to a scanner's findings.
|
|
|
|
Args:
|
|
topic: What to search for (e.g., "flash loan sandwich attack",
|
|
"pump and dump volume spike", "proxy upgrade rug pull")
|
|
chain: Blockchain for context filtering
|
|
address: Token address (included in query for specificity)
|
|
scanner_type: Which scanner is asking (maps to relevant collections)
|
|
max_citations: Max citations to return
|
|
min_similarity: Minimum similarity score
|
|
|
|
Returns:
|
|
List of citation dicts with keys:
|
|
source, title, collection, similarity, snippet, reference
|
|
"""
|
|
try:
|
|
from app.rag_service import search_multi_collection
|
|
except ImportError:
|
|
logger.debug("RAG service not available, skipping citations")
|
|
return []
|
|
|
|
# Build enriched query
|
|
query_parts = [topic]
|
|
if chain and chain.lower() != "general":
|
|
query_parts.append(chain)
|
|
if address:
|
|
query_parts.append(address[:10]) # prefix only, not full address
|
|
enriched_query = " ".join(query_parts)
|
|
|
|
# Get collection priority for this scanner type
|
|
coll_map = SCANNER_COLLECTIONS.get(scanner_type, SCANNER_COLLECTIONS["general"])
|
|
primary = coll_map.get("primary", ["known_scams", "forensic_reports"])
|
|
secondary = coll_map.get("secondary", [])
|
|
|
|
citations = []
|
|
|
|
try:
|
|
# Query primary collections first (higher min_similarity)
|
|
# Timeout: 30s max on first call (model loading), then 10s
|
|
primary_results = await asyncio.wait_for(
|
|
search_multi_collection(
|
|
query=enriched_query,
|
|
collections=primary,
|
|
limit=max_citations,
|
|
min_similarity=min_similarity,
|
|
),
|
|
timeout=30.0,
|
|
)
|
|
|
|
for r in primary_results:
|
|
citation = _format_citation(r, "primary")
|
|
if citation:
|
|
citations.append(citation)
|
|
|
|
# If we need more, dip into secondary collections
|
|
if len(citations) < max_citations and secondary:
|
|
remaining = max_citations - len(citations)
|
|
secondary_results = await asyncio.wait_for(
|
|
search_multi_collection(
|
|
query=enriched_query,
|
|
collections=secondary,
|
|
limit=remaining,
|
|
min_similarity=min_similarity - 0.1, # slightly lower threshold
|
|
),
|
|
timeout=8.0,
|
|
)
|
|
for r in secondary_results:
|
|
citation = _format_citation(r, "secondary")
|
|
if citation:
|
|
citations.append(citation)
|
|
|
|
except (TimeoutError, Exception) as e:
|
|
logger.warning(f"RAG citation query failed for '{topic}': {e}")
|
|
return []
|
|
|
|
# Sort by similarity, deduplicate
|
|
citations.sort(key=lambda x: x.get("similarity", 0), reverse=True)
|
|
seen = set()
|
|
unique = []
|
|
for c in citations:
|
|
key = (c.get("title", ""), c.get("collection", ""))
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique.append(c)
|
|
|
|
return unique[:max_citations]
|
|
|
|
|
|
async def query_address_rag(
|
|
address: str,
|
|
chain: str = "ethereum",
|
|
max_results: int = 5,
|
|
) -> list[dict[str, Any]]:
|
|
"""Quick RAG lookup for a specific address across scam/wallet collections.
|
|
|
|
Used by address_labeler and any scanner that needs to check if an address
|
|
is already known in our RAG databases.
|
|
"""
|
|
try:
|
|
from app.rag_service import search_multi_collection
|
|
except ImportError:
|
|
return []
|
|
|
|
try:
|
|
results = await asyncio.wait_for(
|
|
search_multi_collection(
|
|
query=address,
|
|
collections=["known_scams", "wallet_profiles", "forensic_reports"],
|
|
limit=max_results,
|
|
min_similarity=0.4,
|
|
),
|
|
timeout=10.0,
|
|
)
|
|
formatted = []
|
|
for r in results:
|
|
c = _format_citation(r, "address_lookup")
|
|
if c:
|
|
formatted.append(c)
|
|
return formatted
|
|
except Exception as e:
|
|
logger.debug(f"Address RAG query failed for {address[:10]}...: {e}")
|
|
return []
|
|
|
|
|
|
def _format_citation(raw: dict[str, Any], tier: str) -> dict[str, Any] | None:
|
|
"""Turn a RAG search result into a clean citation dict."""
|
|
if not raw:
|
|
return None
|
|
|
|
# Extract the best text field
|
|
content = raw.get("content", "") or raw.get("text", "") or raw.get("summary", "")
|
|
if not content:
|
|
return None
|
|
|
|
# Build a short snippet (first 200 chars)
|
|
snippet = content[:200].strip()
|
|
if len(content) > 200:
|
|
snippet += "..."
|
|
|
|
# Extract title if available
|
|
title = raw.get("title", "") or raw.get("name", "") or raw.get("id", "")
|
|
if not title:
|
|
# Use first meaningful line
|
|
for line in content.split("\n"):
|
|
line = line.strip()
|
|
if len(line) > 10:
|
|
title = line[:80]
|
|
break
|
|
|
|
# Build reference string
|
|
collection = raw.get("collection", "unknown")
|
|
similarity = raw.get("similarity", 0.0)
|
|
|
|
# Format a human-readable reference
|
|
if collection == "forensic_reports":
|
|
ref_prefix = "Research"
|
|
elif collection == "known_scams":
|
|
ref_prefix = "Known Scam DB"
|
|
elif collection == "wallet_profiles":
|
|
ref_prefix = "Wallet Intel"
|
|
elif collection == "contract_audits":
|
|
ref_prefix = "Audit"
|
|
elif collection == "scam_patterns":
|
|
ref_prefix = "Pattern DB"
|
|
elif collection == "market_intel":
|
|
ref_prefix = "Market Intel"
|
|
else:
|
|
ref_prefix = "RAG"
|
|
|
|
reference = f"[{ref_prefix}]"
|
|
if title:
|
|
reference += f" {title}"
|
|
|
|
return {
|
|
"source": collection,
|
|
"title": title,
|
|
"collection": collection,
|
|
"similarity": round(similarity, 3),
|
|
"snippet": snippet,
|
|
"reference": reference,
|
|
"tier": tier,
|
|
}
|
|
|
|
|
|
def build_citation_string(citations: list[dict[str, Any]], finding: str) -> str:
|
|
"""Build a natural-language citation string for a finding.
|
|
|
|
Example output:
|
|
"Flash loan sandwich pattern detected. Supported by: [Research] Flashot:
|
|
Arbitrage Attack in DeFi (Zhang et al. 2021) [sim: 0.89]; [Known Scam DB]
|
|
0xabc4... flash loan exploit [sim: 0.82]"
|
|
"""
|
|
if not citations:
|
|
return finding
|
|
|
|
parts = [finding, " Supported by:"]
|
|
for i, c in enumerate(citations[:3]): # max 3 inline
|
|
ref = c.get("reference", "RAG")
|
|
sim = c.get("similarity", 0)
|
|
parts.append(f" {ref} [sim: {sim:.2f}]")
|
|
if i < len(citations) - 1 and i < 2:
|
|
parts.append(";")
|
|
|
|
return "".join(parts)
|