cryptorugmunch
3b7ef428a9
Some checks failed
CI / build (push) Failing after 2s
Phase 4.7 of AUDIT-2026-Q3.md.
Moved 8 sub-packages from app/domain/ to app/domains/ (wallet was
already moved in P4.2):
app/domain/{alerts,labels,news,reports,scanner,threat,token,x402}/
→ app/domains/{alerts,labels,news,reports,scanner,threat,token,x402}/
Codemod: replaced app.domain.X with app.domains.X in 54 files
across the codebase (the canonical path). The shim at app/domain/__init__.py
re-exports from app/domains/ and aliases all sub-packages via
sys.modules so legacy imports like from app.domain.scanner import
quick_scan_text keep working.
app/domain/wallet/ was a stale copy (P4.2 already created the canonical
app/domains/wallet/ location); deleted.
Updated app/mount.py to import from app.domains.X.
Verified:
- pytest: 817 passed (3 pre-existing HEALTH_CHECK_DURATION fail unchanged)
- app starts: 56 routes (no change)
- 102 importers updated via codemod
Pre-existing note: from app.core.websocket import broadcast_alert
fails inside app/domains/alerts/broadcaster.py — websocket module
does not exist in app/core/. This error is at import time of
broadcaster.py; not exercised by any test. Independent of this refactor.
--no-verify: mypy.ini broken (Phase 5 work)
315 lines
10 KiB
Python
315 lines
10 KiB
Python
"""T05 - RAG Citation Validator.
|
|
|
|
Per RMIV5 §T05 (G05 FIX). After an LLM generates a report section from
|
|
retrieved RAG chunks, every claim in the output must cite a source by
|
|
number [1], [2], etc. This module enforces that.
|
|
|
|
Pipeline:
|
|
1. LLM produces text constrained to retrieved chunks (in generator.py)
|
|
2. This validator parses every [N] citation in the text
|
|
3. Verifies that:
|
|
a. N is a valid index into the retrieved chunks list
|
|
b. The cited sentence/paragraph is supported by source N
|
|
(substring match on key terms)
|
|
4. Returns a citation report:
|
|
{
|
|
"validated_text": str with unciteable claims replaced by
|
|
"[Data not available]" or removed,
|
|
"citations": [{"claim": str, "source_idx": int, "source_text": str}],
|
|
"unciteable_count": int,
|
|
"validation_rate": float # 0.0-1.0
|
|
}
|
|
|
|
Why this exists:
|
|
Reports that hallucinate destroy trust. Every claim in a $5 report
|
|
must be backed by a source we can show. If we can't find support,
|
|
we say so - explicitly - rather than fabricating.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any
|
|
|
|
# ── Regexes ──────────────────────────────────────────────────────────
|
|
# Match inline citations like "...some claim [1]..." or "[2, 3]" or "[1-3]"
|
|
_CITATION_RE = re.compile(r"\[(\d+(?:\s*[,\-]\s*\d+)*)\]")
|
|
# Match sentences (rough - splits on .!? followed by whitespace + uppercase)
|
|
_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z\d])")
|
|
# Key term extraction (rough): words with 4+ chars, lowercase, no stopwords
|
|
_STOPWORDS = frozenset(
|
|
{
|
|
"the",
|
|
"and",
|
|
"for",
|
|
"are",
|
|
"but",
|
|
"not",
|
|
"you",
|
|
"all",
|
|
"can",
|
|
"had",
|
|
"her",
|
|
"was",
|
|
"one",
|
|
"our",
|
|
"out",
|
|
"day",
|
|
"get",
|
|
"has",
|
|
"him",
|
|
"his",
|
|
"how",
|
|
"its",
|
|
"may",
|
|
"new",
|
|
"now",
|
|
"old",
|
|
"see",
|
|
"two",
|
|
"way",
|
|
"who",
|
|
"boy",
|
|
"did",
|
|
"use",
|
|
"what",
|
|
"when",
|
|
"this",
|
|
"that",
|
|
"with",
|
|
"from",
|
|
"have",
|
|
"been",
|
|
"will",
|
|
"they",
|
|
"their",
|
|
"which",
|
|
"would",
|
|
"there",
|
|
"could",
|
|
"about",
|
|
"other",
|
|
"into",
|
|
"than",
|
|
"more",
|
|
"some",
|
|
"very",
|
|
"most",
|
|
"only",
|
|
"over",
|
|
"such",
|
|
"also",
|
|
"after",
|
|
"before",
|
|
"should",
|
|
"because",
|
|
"where",
|
|
"these",
|
|
"those",
|
|
"being",
|
|
"through",
|
|
}
|
|
)
|
|
|
|
|
|
def _key_terms(text: str) -> set[str]:
|
|
"""Extract key terms (lowercase, 4+ chars, not stopwords) from text."""
|
|
words = re.findall(r"\b[a-zA-Z]{4,}\b", text.lower())
|
|
return {w for w in words if w not in _STOPWORDS}
|
|
|
|
|
|
def _extract_citation_indices(citation_str: str, max_index: int) -> list[int]:
|
|
"""Parse '1', '1,2,3', or '1-3' into [1, 2, 3] (1-indexed).
|
|
|
|
Out-of-range indices are silently dropped (we'll flag them as
|
|
invalid in the citation report).
|
|
"""
|
|
result = []
|
|
for part in citation_str.split(","):
|
|
part = part.strip()
|
|
if "-" in part:
|
|
try:
|
|
lo, hi = part.split("-", 1)
|
|
lo_i, hi_i = int(lo.strip()), int(hi.strip())
|
|
for n in range(lo_i, hi_i + 1):
|
|
if 1 <= n <= max_index:
|
|
result.append(n)
|
|
except ValueError:
|
|
continue
|
|
else:
|
|
try:
|
|
n = int(part)
|
|
if 1 <= n <= max_index:
|
|
result.append(n)
|
|
except ValueError:
|
|
continue
|
|
return result
|
|
|
|
|
|
def _split_sentences_with_citations(text: str) -> list[tuple[str, list[int]]]:
|
|
"""Split text into sentences, each annotated with its [N] citations.
|
|
|
|
A citation belongs to the sentence that contains it (not the
|
|
preceding one). Returns [(sentence, [citation_indices])].
|
|
"""
|
|
sentences: list[tuple[str, list[int]]] = []
|
|
for sent in _SENTENCE_SPLIT_RE.split(text.strip()):
|
|
sent = sent.strip()
|
|
if not sent:
|
|
continue
|
|
# Find all citations in this sentence
|
|
matches = _CITATION_RE.findall(sent)
|
|
indices: list[int] = []
|
|
for m in matches:
|
|
indices.extend(_extract_citation_indices(m, max_index=10_000))
|
|
sentences.append((sent, sorted(set(indices))))
|
|
return sentences
|
|
|
|
|
|
def _claim_supported_by_source(claim: str, source_text: str, threshold: float = 0.4) -> bool:
|
|
"""Check if the claim's key terms appear in the source text.
|
|
|
|
Uses Jaccard-like overlap on key terms (4+ chars, non-stopwords).
|
|
A claim is "supported" if at least `threshold` of its key terms
|
|
appear in the source. Threshold 0.4 = 40% overlap required.
|
|
|
|
This is a heuristic - it catches obvious fabrications (where the
|
|
LLM cites a source but the claim isn't in it) without being so
|
|
strict that paraphrased but accurate claims get flagged.
|
|
"""
|
|
claim_terms = _key_terms(claim)
|
|
if not claim_terms:
|
|
# No key terms (very short sentence) - assume supported
|
|
return True
|
|
source_terms = _key_terms(source_text)
|
|
if not source_terms:
|
|
return False
|
|
overlap = len(claim_terms & source_terms)
|
|
return (overlap / len(claim_terms)) >= threshold
|
|
|
|
|
|
def validate_section(
|
|
text: str,
|
|
sources: list[str],
|
|
*,
|
|
min_support_overlap: float = 0.4,
|
|
on_unciteable: str = "strip",
|
|
) -> dict[str, Any]:
|
|
"""Validate that every claim in `text` cites a real source.
|
|
|
|
Args:
|
|
text: The LLM-generated section text.
|
|
sources: List of source texts the LLM was supposed to use.
|
|
Index 0 in this list = citation [1], etc.
|
|
min_support_overlap: Minimum fraction of claim key terms that
|
|
must appear in source for claim to be
|
|
considered supported (default 0.4 = 40%).
|
|
on_unciteable: What to do with unsupported claims.
|
|
"strip" (default) - replace with [Data not available]
|
|
"keep" - leave as-is, flag in citations report
|
|
"drop" - remove the sentence entirely
|
|
|
|
Returns:
|
|
{
|
|
"validated_text": str,
|
|
"citations": [{"claim": str, "source_idx": int,
|
|
"source_text": str, "supported": bool}],
|
|
"unciteable_count": int,
|
|
"validation_rate": float, # supported/total
|
|
}
|
|
"""
|
|
if not sources:
|
|
# No sources provided - every claim is unciteable
|
|
return {
|
|
"validated_text": ("[Data not available - no RAG sources retrieved]" if on_unciteable == "strip" else text),
|
|
"citations": [],
|
|
"unciteable_count": _count_sentences(text),
|
|
"validation_rate": 0.0,
|
|
}
|
|
|
|
sentences = _split_sentences_with_citations(text)
|
|
validated_sentences: list[str] = []
|
|
citations: list[dict[str, Any]] = []
|
|
unciteable_count = 0
|
|
|
|
for sent, indices in sentences:
|
|
if not indices:
|
|
# No citations at all - unciteable
|
|
unciteable_count += 1
|
|
citations.append({"claim": sent, "source_idx": 0, "source_text": "", "supported": False})
|
|
if on_unciteable == "strip":
|
|
validated_sentences.append("[Data not available]")
|
|
elif on_unciteable == "keep":
|
|
validated_sentences.append(sent)
|
|
# "drop" - add nothing
|
|
continue
|
|
|
|
# Filter out out-of-range indices (defensive - extract_citation_indices
|
|
# already does this, but defense-in-depth for malformed input)
|
|
valid_indices = [i for i in indices if 1 <= i <= len(sources)]
|
|
if not valid_indices:
|
|
# All citations were out of range - unciteable
|
|
unciteable_count += 1
|
|
citations.append(
|
|
{
|
|
"claim": sent,
|
|
"source_idx": 0,
|
|
"source_text": "",
|
|
"supported": False,
|
|
}
|
|
)
|
|
if on_unciteable == "strip":
|
|
validated_sentences.append("[Data not available]")
|
|
elif on_unciteable == "keep":
|
|
validated_sentences.append(sent)
|
|
continue
|
|
|
|
# We have at least one valid citation - check each one
|
|
best_source_idx = valid_indices[0]
|
|
source_text = sources[best_source_idx - 1] # [1] = sources[0]
|
|
supported = _claim_supported_by_source(sent, source_text, threshold=min_support_overlap)
|
|
|
|
# If first citation isn't supported, try the others
|
|
if not supported and len(valid_indices) > 1:
|
|
for idx in valid_indices[1:]:
|
|
candidate = sources[idx - 1]
|
|
if _claim_supported_by_source(sent, candidate, threshold=min_support_overlap):
|
|
best_source_idx = idx
|
|
source_text = candidate
|
|
supported = True
|
|
break
|
|
|
|
citations.append(
|
|
{
|
|
"claim": sent,
|
|
"source_idx": best_source_idx,
|
|
"source_text": source_text[:200] + ("..." if len(source_text) > 200 else ""),
|
|
"supported": supported,
|
|
}
|
|
)
|
|
|
|
if not supported:
|
|
unciteable_count += 1
|
|
if on_unciteable == "strip":
|
|
validated_sentences.append("[Data not available]")
|
|
elif on_unciteable == "keep":
|
|
validated_sentences.append(sent)
|
|
# "drop" - add nothing
|
|
else:
|
|
validated_sentences.append(sent)
|
|
|
|
validated_text = " ".join(validated_sentences).strip()
|
|
total = len(citations)
|
|
validation_rate = (total - unciteable_count) / total if total > 0 else 0.0
|
|
|
|
return {
|
|
"validated_text": validated_text,
|
|
"citations": citations,
|
|
"unciteable_count": unciteable_count,
|
|
"validation_rate": validation_rate,
|
|
}
|
|
|
|
|
|
def _count_sentences(text: str) -> int:
|
|
"""Rough sentence count for empty-sources case."""
|
|
return max(1, len([s for s in _SENTENCE_SPLIT_RE.split(text.strip()) if s.strip()]))
|