merge: chore/cleanup-remove-bloat-and-secrets into main

app/domain/news/__init__.py

@@ -0,0 +1,5 @@
+"""T28 News Intelligence — thin HTTP layer."""
+from .admin_router import router as admin_router
+from .router import router
+
+__all__ = ["admin_router", "router"]

app/domain/news/admin_router.py

@@ -0,0 +1,14 @@
+"""T28 RSS Ingest HTTP endpoint."""
+
+from fastapi import APIRouter
+
+from app.domain.news.ingest import ingest_all
+
+router = APIRouter(prefix="/api/v1/news/_admin", tags=["news-admin"])
+
+
+@router.post("/ingest")
+async def trigger_ingest() -> dict:
+    """Trigger RSS ingest now (synchronous). Returns counts."""
+    result = await ingest_all()
+    return result

app/domain/news/clusterer.py

@@ -0,0 +1,415 @@
+"""T03 — News story clustering (G04 FIX).
+
+Per MINIMAX_M3_TASKS.md T03. MinHash + DBSCAN dedupes raw RSS items into
+single "stories" so AI agents don't see the same CoinDesk/The Block story
+counted 2-3x in their signal.
+
+Algorithm:
+  1. MinHash signature (128 permutations) on shingled title+body (first 500 chars)
+  2. DBSCAN clusters within 30-minute windows, Jaccard threshold 0.6, eps=0.15
+  3. Each cluster = one story with all source URLs, sentiment avg, item count
+  4. Persist clusters to Postgres `news_clusters` table; raw items unchanged
+
+Endpoints:
+  GET /api/v1/news?clustered=true    returns stories (clusters), not raw items
+  GET /api/v1/news                   raw items (legacy)
+
+This module is pure logic — no I/O at import time. Router/background job
+call `cluster_items(items) -> list[StoryCluster]`.
+"""
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+import time
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+
+logger = logging.getLogger(__name__)
+
+
+# ── Tokenization ────────────────────────────────────────────────────
+_TOKEN_RE = re.compile(r"[a-z0-9]{3,}", re.IGNORECASE)
+
+
+def _shingles(text: str, k: int = 3) -> set[str]:
+    """k-shingle set of lowercased alphanumeric tokens. For Jaccard."""
+    if not text:
+        return set()
+    toks = _TOKEN_RE.findall(text.lower())
+    if len(toks) < k:
+        return set(toks)
+    return {" ".join(toks[i : i + k]) for i in range(len(toks) - k + 1)}
+
+
+# ── MinHash ─────────────────────────────────────────────────────────
+_NUM_PERM = 128
+_MAX_HASH = (1 << 32) - 1
+
+
+def _minhash_signature(shingles: set[str], seed: int = 42) -> list[int]:
+    """128-permutation MinHash signature of a shingle set.
+
+    Uses SHA-256 seeded permutations — fast, deterministic, no numpy.
+    """
+    if not shingles:
+        return [_MAX_HASH] * _NUM_PERM
+    sig: list[int] = []
+    for i in range(_NUM_PERM):
+        m = _MAX_HASH
+        for s in shingles:
+            h = int.from_bytes(
+                hashlib.sha256(f"{i}:{seed}:{s}".encode()).digest()[:4],
+                "big",
+            )
+            if h < m:
+                m = h
+        sig.append(m)
+    return sig
+
+
+def _jaccard_minhash(a: list[int], b: list[int]) -> float:
+    """Estimate Jaccard similarity from two MinHash signatures."""
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    return sum(1 for x, y in zip(a, b, strict=False) if x == y) / len(a)
+
+
+# ── DBSCAN (pure-python, no sklearn dep) ────────────────────────────
+def _dbscan(
+    signatures: list[list[int]],
+    eps: float = 0.4,
+    min_samples: int = 2,
+) -> list[int]:
+    """Density-based clustering. Returns cluster id per item (-1 = noise).
+
+    Similarity is Jaccard (estimated via MinHash). Neighbours are pairs
+    with Jaccard distance <= eps (i.e. similarity >= 1 - eps).
+    Default eps=0.4 means similarity >= 0.6 (per T03 spec).
+    """
+    n = len(signatures)
+    labels = [-1] * n
+    cluster_id = 0
+    for i in range(n):
+        if labels[i] != -1:
+            continue
+        neighbors = [
+            j
+            for j in range(n)
+            if i != j and (1.0 - _jaccard_minhash(signatures[i], signatures[j])) <= eps
+        ]
+        if len(neighbors) < min_samples - 1:
+            # not enough neighbours — mark as noise (may become border later)
+            continue
+        labels[i] = cluster_id
+        seed_set = list(neighbors)
+        k = 0
+        while k < len(seed_set):
+            q = seed_set[k]
+            if labels[q] == -1:
+                labels[q] = cluster_id
+                q_neighbors = [
+                    j
+                    for j in range(n)
+                    if j != q
+                    and (1.0 - _jaccard_minhash(signatures[q], signatures[j])) <= eps
+                ]
+                if len(q_neighbors) >= min_samples - 1:
+                    seed_set.extend(q_neighbors)
+            elif labels[q] is None or labels[q] == -1:
+                labels[q] = cluster_id
+            k += 1
+        cluster_id += 1
+    return labels
+
+
+# ── Domain types ────────────────────────────────────────────────────
+@dataclass
+class NewsItem:
+    """Minimal news item for clustering. Adapts from DB rows or dicts."""
+
+    id: str
+    title: str
+    body: str = ""
+    source: str = ""
+    url: str = ""
+    published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
+    sentiment: float = 0.0
+
+    @classmethod
+    def from_row(cls, row: dict) -> NewsItem:
+        published = row.get("published_at") or row.get("created_at")
+        if isinstance(published, str):
+            try:
+                published = datetime.fromisoformat(published.replace("Z", "+00:00"))
+            except (ValueError, AttributeError):
+                published = datetime.now(UTC)
+        elif not isinstance(published, datetime):
+            published = datetime.now(UTC)
+        return cls(
+            id=str(row.get("id", row.get("news_id", ""))),
+            title=row.get("title", "") or "",
+            body=(row.get("body") or row.get("summary") or "")[:500],
+            source=row.get("source", "") or "",
+            url=row.get("url", "") or "",
+            published_at=published,
+            sentiment=float(row.get("sentiment", 0.0) or 0.0),
+        )
+
+
+@dataclass
+class StoryCluster:
+    """One deduplicated story spanning 1+ source items."""
+
+    cluster_id: str
+    representative_title: str
+    source_urls: list[str]
+    sources: list[str]
+    first_seen: datetime
+    last_updated: datetime
+    item_count: int
+    sentiment_avg: float
+    item_ids: list[str]
+
+    def to_dict(self) -> dict:
+        return {
+            "cluster_id": self.cluster_id,
+            "representative_title": self.representative_title,
+            "source_urls": self.source_urls,
+            "sources": self.sources,
+            "first_seen": self.first_seen.isoformat(),
+            "last_updated": self.last_updated.isoformat(),
+            "item_count": self.item_count,
+            "sentiment_avg": round(self.sentiment_avg, 3),
+            "item_ids": self.item_ids,
+        }
+
+
+# ── Main entry point ────────────────────────────────────────────────
+def cluster_items(
+    items: list[NewsItem],
+    window_minutes: int = 30,
+    eps: float = 0.4,
+    min_samples: int = 2,
+) -> list[StoryCluster]:
+    """Cluster news items into stories.
+
+    Items are first grouped by 30-minute time windows, then DBSCAN runs
+    on MinHash signatures within each window. Single-item clusters are
+    kept (they're "noise" in DBSCAN terms but valid singleton stories).
+
+    `eps` is the Jaccard DISTANCE threshold (1 - similarity). Per the
+    task spec, two items cluster together when Jaccard similarity >= 0.6,
+    so distance <= 0.4, so eps=0.4. Tighten for stricter clusters.
+    """
+    t0 = time.time()
+    if not items:
+        return []
+
+    # Group by time window
+    windows: dict[datetime, list[NewsItem]] = {}
+    for it in sorted(items, key=lambda x: x.published_at):
+        bucket = it.published_at.replace(
+            minute=(it.published_at.minute // window_minutes) * window_minutes,
+            second=0,
+            microsecond=0,
+        )
+        windows.setdefault(bucket, []).append(it)
+
+    stories: list[StoryCluster] = []
+    for _bucket, group in windows.items():
+        if len(group) == 1:
+            # singleton — still a story
+            it = group[0]
+            stories.append(
+                StoryCluster(
+                    cluster_id=hashlib.sha1(
+                        f"single:{it.id}:{it.published_at.isoformat()}".encode()
+                    ).hexdigest()[:16],
+                    representative_title=it.title,
+                    source_urls=[it.url] if it.url else [],
+                    sources=[it.source] if it.source else [],
+                    first_seen=it.published_at,
+                    last_updated=it.published_at,
+                    item_count=1,
+                    sentiment_avg=it.sentiment,
+                    item_ids=[it.id],
+                )
+            )
+            continue
+
+        sigs = [_minhash_signature(_shingles(f"{it.title} {it.body}")) for it in group]
+        labels = _dbscan(sigs, eps=eps, min_samples=min_samples)
+        # Singletons (label == -1) still become stories
+        clusters: dict[int, list[int]] = {}
+        for idx, lbl in enumerate(labels):
+            clusters.setdefault(lbl if lbl != -1 else idx, []).append(idx)
+        for _cid, indices in clusters.items():
+            members = [group[i] for i in indices]
+            # Pick representative = longest title (usually the most descriptive)
+            rep = max(members, key=lambda x: len(x.title))
+            sentiments = [m.sentiment for m in members if m.sentiment is not None]
+            avg_sent = sum(sentiments) / len(sentiments) if sentiments else 0.0
+            cluster_id = hashlib.sha1(
+                ":".join(sorted(m.id for m in members)).encode()
+            ).hexdigest()[:16]
+            stories.append(
+                StoryCluster(
+                    cluster_id=cluster_id,
+                    representative_title=rep.title,
+                    source_urls=[m.url for m in members if m.url],
+                    sources=sorted({m.source for m in members if m.source}),
+                    first_seen=min(m.published_at for m in members),
+                    last_updated=max(m.published_at for m in members),
+                    item_count=len(members),
+                    sentiment_avg=avg_sent,
+                    item_ids=[m.id for m in members],
+                )
+            )
+
+    logger.info(
+        "news_clustered items=%d stories=%d windows=%d elapsed_ms=%.1f",
+        len(items),
+        len(stories),
+        len(windows),
+        (time.time() - t0) * 1000,
+    )
+    return stories
+
+
+# ── DB persistence (optional, lazy import) ──────────────────────────
+_PG_SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS news_clusters (
+  cluster_id           TEXT PRIMARY KEY,
+  representative_title TEXT NOT NULL,
+  first_seen           TIMESTAMPTZ NOT NULL,
+  last_updated         TIMESTAMPTZ NOT NULL,
+  item_count           INTEGER NOT NULL DEFAULT 0,
+  sentiment_avg        DOUBLE PRECISION NOT NULL DEFAULT 0.0,
+  source_urls          JSONB NOT NULL DEFAULT '[]'::jsonb,
+  sources              JSONB NOT NULL DEFAULT '[]'::jsonb,
+  item_ids             JSONB NOT NULL DEFAULT '[]'::jsonb,
+  created_at           TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+CREATE INDEX IF NOT EXISTS news_clusters_last_updated_idx
+  ON news_clusters (last_updated DESC);
+"""
+
+
+async def ensure_schema() -> bool:
+    """Create news_clusters table if missing. Returns True on success."""
+    try:
+        import asyncpg
+
+        from app.core.db_pool import PG_URL
+
+        conn = await asyncpg.connect(PG_URL)
+        try:
+            await conn.execute(_PG_SCHEMA_SQL)
+        finally:
+            await conn.close()
+        logger.info("news_clusters_schema_ready")
+        return True
+    except Exception as exc:
+        logger.warning("news_clusters_schema_failed err=%s", exc)
+        return False
+
+
+async def persist_clusters(stories: list[StoryCluster]) -> int:
+    """Upsert stories to Postgres. Returns rows affected."""
+    if not stories:
+        return 0
+    try:
+        import json
+
+        import asyncpg
+
+        from app.core.db_pool import PG_URL
+
+        conn = await asyncpg.connect(PG_URL)
+        try:
+            rows = [
+                (
+                    s.cluster_id,
+                    s.representative_title,
+                    s.first_seen,
+                    s.last_updated,
+                    s.item_count,
+                    s.sentiment_avg,
+                    json.dumps(s.source_urls),
+                    json.dumps(s.sources),
+                    json.dumps(s.item_ids),
+                )
+                for s in stories
+            ]
+            await conn.executemany(
+                """
+                INSERT INTO news_clusters
+                  (cluster_id, representative_title, first_seen, last_updated,
+                   item_count, sentiment_avg, source_urls, sources, item_ids)
+                VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9::jsonb)
+                ON CONFLICT (cluster_id) DO UPDATE SET
+                  representative_title = EXCLUDED.representative_title,
+                  first_seen = EXCLUDED.first_seen,
+                  last_updated = EXCLUDED.last_updated,
+                  item_count = EXCLUDED.item_count,
+                  sentiment_avg = EXCLUDED.sentiment_avg,
+                  source_urls = EXCLUDED.source_urls,
+                  sources = EXCLUDED.sources,
+                  item_ids = EXCLUDED.item_ids
+                """,
+                rows,
+            )
+            return len(rows)
+        finally:
+            await conn.close()
+    except Exception as exc:
+        logger.warning("news_clusters_persist_failed err=%s", exc)
+        return 0
+
+
+async def load_recent_clusters(limit: int = 50) -> list[dict]:
+    """Load recent clusters from Postgres."""
+    try:
+        import json
+
+        import asyncpg
+
+        from app.core.db_pool import PG_URL
+
+        conn = await asyncpg.connect(PG_URL)
+        try:
+            rows = await conn.fetch(
+                "SELECT * FROM news_clusters ORDER BY last_updated DESC LIMIT $1",
+                limit,
+            )
+            return [
+                {
+                    "cluster_id": r["cluster_id"],
+                    "representative_title": r["representative_title"],
+                    "first_seen": r["first_seen"].isoformat(),
+                    "last_updated": r["last_updated"].isoformat(),
+                    "item_count": r["item_count"],
+                    "sentiment_avg": r["sentiment_avg"],
+                    "source_urls": json.loads(r["source_urls"]),
+                    "sources": json.loads(r["sources"]),
+                    "item_ids": json.loads(r["item_ids"]),
+                }
+                for r in rows
+            ]
+        finally:
+            await conn.close()
+    except Exception as exc:
+        logger.warning("news_clusters_load_failed err=%s", exc)
+        return []
+
+
+__all__ = [
+    "NewsItem",
+    "StoryCluster",
+    "cluster_items",
+    "ensure_schema",
+    "load_recent_clusters",
+    "persist_clusters",
+]

app/domain/news/ingest.py

@@ -0,0 +1,218 @@
+"""T28 RSS Ingest — populates news_items + crypto_news from RSS feeds.
+
+Sources (v4.0 master stack): 5+ RSS feeds
+- CoinDesk
+- Cointelegraph
+- The Block
+- Decrypt
+- BeInCrypto
+
+Caches raw HTML to MinIO, writes metadata to news_items,
+embeds into RAG engine for semantic search.
+
+Run as a cron: every 15 minutes.
+"""
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import hashlib
+import logging
+from datetime import UTC, datetime
+
+import feedparser
+import httpx
+from pydantic import HttpUrl
+
+from app.catalog.models import NewsItem, utcnow
+from app.catalog.service import get_catalog
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+log = logging.getLogger(__name__)
+
+# Default feeds (per v4.0 §T28)
+DEFAULT_FEEDS: list[dict[str, str]] = [
+    {"name": "coindesk", "url": "https://www.coindesk.com/arc/outboundfeeds/rss/"},
+    {"name": "cointelegraph", "url": "https://cointelegraph.com/rss"},
+    {"name": "theblock", "url": "https://www.theblock.co/rss.xml"},
+    {"name": "decrypt", "url": "https://decrypt.co/feed"},
+    {"name": "beincrypto", "url": "https://beincrypto.com/feed/"},
+]
+
+
+def _stable_id(source: str, url: str) -> str:
+    """Stable news_id from source + URL."""
+    h = hashlib.md5(f"{source}:{url}".encode()).hexdigest()[:24]
+    return f"{source}:{h}"
+
+
+def _detect_chains(text: str) -> list[str]:
+    """Heuristic chain detection from text content."""
+    text_l = text.lower()
+    chains = []
+    chain_map = {
+        "solana": ["solana", "sol ", "$sol"],
+        "ethereum": ["ethereum", "eth ", "$eth", "ether"],
+        "bitcoin": ["bitcoin", "btc ", "$btc"],
+        "base": ["base ", "basechain", "coinbase base"],
+        "arbitrum": ["arbitrum", "arb "],
+        "polygon": ["polygon", "matic"],
+        "bsc": ["bsc", "bnb chain", "binance smart chain"],
+        "tron": ["tron", "trx "],
+        "avalanche": ["avalanche", "avax"],
+    }
+    for chain, keywords in chain_map.items():
+        if any(k in text_l for k in keywords):
+            chains.append(chain)
+    return chains
+
+
+def _extract_tickers(text: str) -> list[str]:
+    """Extract $TICKER style mentions."""
+    import re
+
+    return list(set(re.findall(r"\$([A-Z]{2,6})\b", text)))
+
+
+async def fetch_feed(client: httpx.AsyncClient, feed: dict[str, str]) -> list[dict]:
+    """Fetch and parse a single RSS feed."""
+    try:
+        r = await client.get(feed["url"], timeout=10.0, follow_redirects=True)
+        r.raise_for_status()
+        parsed = feedparser.parse(r.content)
+        items = []
+        for entry in parsed.entries[:30]:  # cap per feed
+            items.append(
+                {
+                    "source": feed["name"],
+                    "title": entry.get("title", ""),
+                    "url": entry.get("link", ""),
+                    "summary": entry.get("summary", "")[:2000],
+                    "published": entry.get("published_parsed") or entry.get("updated_parsed"),
+                }
+            )
+        return items
+    except Exception as e:
+        log.warning("feed_fetch_fail name=%s err=%s", feed["name"], e)
+        return []
+
+
+async def ingest_all(
+    feeds: list[dict[str, str]] | None = None,
+    embed: bool = True,
+    collection: str = "news_articles",
+) -> dict:
+    """Ingest from all configured RSS feeds. Returns counts."""
+    feeds = feeds or DEFAULT_FEEDS
+    cat = get_catalog()
+    await cat._init_stores()
+    if not cat._health.postgres:
+        return {"error": "postgres unavailable"}
+
+    counts = {"feeds_ok": 0, "feeds_fail": 0, "items": 0, "ingested": 0, "duplicate": 0, "errors": 0}
+    async with httpx.AsyncClient() as client:
+        # Fetch all feeds in parallel
+        results = await asyncio.gather(
+            *[fetch_feed(client, f) for f in feeds], return_exceptions=True
+        )
+    items_to_ingest: list[NewsItem] = []
+    for _i, r in enumerate(results):
+        if isinstance(r, Exception) or not r:
+            counts["feeds_fail"] += 1
+            continue
+        counts["feeds_ok"] += 1
+        for entry in r:
+            counts["items"] += 1
+            try:
+                pub_dt = utcnow()
+                if entry.get("published"):
+                    with contextlib.suppress(Exception):
+                        pub_dt = datetime(*entry["published"][:6], tzinfo=UTC)
+                text = f"{entry['title']} {entry['summary']}"
+                chains = _detect_chains(text)
+                tickers = _extract_tickers(text)
+                news_id = _stable_id(entry["source"], entry["url"])
+                ni = NewsItem(
+                    news_id=news_id,
+                    url=HttpUrl(entry["url"]) if entry["url"] else HttpUrl("https://unknown.local"),
+                    title=entry["title"][:500],
+                    summary=entry["summary"][:2000],
+                    body_markdown=None,
+                    source=entry["source"],
+                    published_at=pub_dt,
+                    ingested_at=utcnow(),
+                    chains_mentioned=chains,
+                    tokens_mentioned=tickers,
+                )
+                items_to_ingest.append(ni)
+            except Exception as e:
+                counts["errors"] += 1
+                log.debug("item_build_fail: %s", e)
+
+    # Save to Postgres
+    if cat._health.postgres and items_to_ingest:
+        try:
+            async with cat._pg_pool.acquire() as conn:
+                for ni in items_to_ingest:
+                    try:
+                        # Check if exists
+                        existing = await conn.fetchval(
+                            "SELECT 1 FROM news_items WHERE news_id=$1", ni.news_id
+                        )
+                        if existing:
+                            counts["duplicate"] += 1
+                            continue
+                        # Insert
+                        await conn.execute(
+                            """
+                            INSERT INTO news_items (
+                                news_id, url, title, summary, body_markdown,
+                                source, published_at, ingested_at,
+                                chains_mentioned, tokens_mentioned, sentiment_score
+                            ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
+                            """,
+                            ni.news_id, str(ni.url), ni.title, ni.summary, ni.body_markdown,
+                            ni.source, ni.published_at, ni.ingested_at,
+                            ni.chains_mentioned, ni.tokens_mentioned, ni.sentiment_score,
+                        )
+                        counts["ingested"] += 1
+                    except Exception as e:
+                        counts["errors"] += 1
+                        log.debug("item_insert_fail: %s", e)
+        except Exception as e:
+            log.warning("ingest_postgres_fail: %s", e)
+
+    # Embed into RAG for semantic search
+    if embed and items_to_ingest:
+        try:
+            for ni in items_to_ingest[:50]:  # cap RAG embeds
+                r = await cat.rag_ingest(
+                    content=f"{ni.title}\n{ni.summary}",
+                    collection=collection,
+                    doc_id=ni.news_id,
+                    metadata={"source": ni.source, "news_id": ni.news_id},
+                )
+                if r.get("status") == "ok" and r.get("qdrant_point_id"):
+                    # Update news_items with rag_embedding_id
+                    try:
+                        async with cat._pg_pool.acquire() as conn:
+                            await conn.execute(
+                                "UPDATE news_items SET rag_embedding_id=$1 WHERE news_id=$2",
+                                r["qdrant_point_id"], ni.news_id,
+                            )
+                    except Exception:
+                        pass
+        except Exception as e:
+            log.warning("ingest_rag_fail: %s", e)
+
+    return counts
+
+
+# ── CLI entry point ──────────────────────────────────────────────
+if __name__ == "__main__":
+    import json
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+    result = asyncio.run(ingest_all())
+    logger.info(json.dumps(result, indent=2))

app/domain/news/router.py

@@ -0,0 +1,450 @@
+"""T28 News Intelligence Product.
+
+Per v4.0 §T28. Four endpoints surface the news pipeline as a product:
+  POST /api/v1/news              list, paginated, filterable
+  GET  /api/v1/news/trending     time-decay weighted
+  GET  /api/v1/news/{news_id}    single item
+  POST /api/v1/news/{news_id}/analyze  LLM analysis (LiteLLM)
+
+Data sources:
+  - crypto_news (legacy table, 1750+ items from RSS feeds)
+  - news_items (new catalog table, populated by RSS ingest)
+  - Qdrant embeddings for semantic search (rag_embedding_id)
+
+Time-decay scoring for /trending:
+  score = recency_decay * source_authority * social_velocity * abs(sentiment)
+  recency_decay = 0.5 ** (hours_old / 6)        half-life 6h
+  source_authority: tier-1 (CoinDesk, The Block) = 1.0, tier-2 = 0.5, tier-3 = 0.2
+  social_velocity: tweets/shares in last hour (1 + n/100, capped at 2x)
+  abs(sentiment): polarizing news ranks higher
+"""
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.catalog.llm_router import LLMRouter
+from app.catalog.models import utcnow
+from app.catalog.service import get_catalog
+
+router = APIRouter(prefix="/api/v1/news", tags=["news"])
+
+
+# ── Time-decay scoring (per v4.0 §T28) ────────────────────────────
+SOURCE_AUTHORITY: dict[str, float] = {
+    # Tier 1 — major crypto-native outlets
+    "coindesk": 1.0,
+    "the block": 1.0,
+    "decrypt": 1.0,
+    "cointelegraph": 1.0,
+    # Tier 2 — solid crypto coverage
+    "beincrypto": 0.7,
+    "u.today": 0.7,
+    "crypto.news": 0.7,
+    "blockworks": 0.8,
+    # Tier 3 — general / RSS aggregators
+    "google-crypto": 0.5,
+    "reddit-crypto": 0.4,
+    "twitter-crypto": 0.3,
+}
+
+
+def recency_decay(hours_old: float, half_life: float = 6.0) -> float:
+    """Exponential decay: 1.0 at 0h, 0.5 at half_life, 0.25 at 2*half_life."""
+    if hours_old < 0:
+        return 1.0
+    return 0.5 ** (hours_old / half_life)
+
+
+def source_authority(source: str) -> float:
+    s = (source or "").lower().strip()
+    for key, val in SOURCE_AUTHORITY.items():
+        if key in s:
+            return val
+    return 0.3  # unknown source
+
+
+def trend_score(
+    hours_old: float, source: str, sentiment: float | None, social_velocity: int = 0
+) -> float:
+    """Composite trending score per v4.0 formula."""
+    s_auth = source_authority(source)
+    s_vel = min(2.0, 1.0 + social_velocity / 100.0)
+    s_sent = 1.0 + abs(sentiment or 0.0)
+    return recency_decay(hours_old) * s_auth * s_vel * s_sent
+
+
+# ── Response models ────────────────────────────────────────────────
+class NewsItemOut(BaseModel):
+    model_config = ConfigDict(strict=False)  # accept None for any field with default
+
+    news_id: str | None = None
+    url: str | None = None
+    title: str | None = None
+    summary: str | None = ""
+    source: str | None = "unknown"
+    published_at: datetime | None = None
+    chains_mentioned: list[str] = Field(default_factory=list)
+    tokens_mentioned: list[str] = Field(default_factory=list)
+    sentiment_score: float | None = None
+    score: float | None = None  # only set for /trending
+
+
+class NewsListResponse(BaseModel):
+    items: list[NewsItemOut]
+    total: int
+    offset: int
+
+
+class NewsAnalysisResponse(BaseModel):
+    news_id: str
+    analysis: str | None
+    model: str | None = None
+    error: str | None = None
+
+
+# ── Row adapter (legacy crypto_news → NewsItemOut) ───────────────
+def _adapt_legacy_row(row: dict) -> NewsItemOut:
+    """Convert a crypto_news row to NewsItemOut shape."""
+    # published is a text field in legacy; try ISO parse
+    pub_str = row.get("published") or row.get("ingested_at")
+    pub_dt = utcnow()
+    if pub_str:
+        try:
+            if isinstance(pub_str, (int, float)):
+                pub_dt = datetime.fromtimestamp(float(pub_str), tz=UTC)
+            else:
+                # Try common formats
+                for fmt in (
+                    "%Y-%m-%dT%H:%M:%S.%fZ",
+                    "%Y-%m-%dT%H:%M:%SZ",
+                    "%Y-%m-%dT%H:%M:%S",
+                    "%Y-%m-%d %H:%M:%S",
+                ):
+                    try:
+                        pub_dt = datetime.strptime(str(pub_str)[:19], fmt).replace(tzinfo=UTC)
+                        break
+                    except ValueError:
+                        continue
+        except Exception:
+            pass
+    return NewsItemOut(
+        news_id=row.get("id", ""),
+        url=row.get("url", ""),
+        title=row.get("title", ""),
+        summary=(row.get("content") or "")[:500],
+        source=row.get("source", "unknown"),
+        published_at=pub_dt,
+        chains_mentioned=[],
+        tokens_mentioned=row.get("tickers") or [],
+        sentiment_score=row.get("sentiment"),
+    )
+
+
+# ── POST /api/v1/news (list with filters) ─────────────────────────
+@router.post("", response_model=NewsListResponse)
+async def list_news(
+    chain: str | None = None,
+    token: str | None = None,
+    category: str | None = None,
+    since_hours: int = Query(24, ge=1, le=720),
+    limit: int = Query(20, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    sort: str = Query("recency", pattern="^(recency|relevance|sentiment)$"),
+    clustered: bool = Query(False, description="T03: dedupe via MinHash+DBSCAN into stories"),
+) -> NewsListResponse:
+    """List news items with filters. Reads from both news_items (new) and crypto_news (legacy)."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    items: list[NewsItemOut] = []
+
+    # New table
+    if catalog._health.postgres:
+        try:
+            cutoff = utcnow() - timedelta(hours=since_hours)
+            query = "SELECT news_id, url, title, summary, source, published_at, sentiment_score, chains_mentioned, tokens_mentioned FROM news_items WHERE published_at > $1"
+            params: list[Any] = [cutoff]
+            if chain:
+                query += f" AND ${len(params)+1} = ANY(chains_mentioned)"
+                params.append(chain)
+            if token:
+                query += f" AND ${len(params)+1} = ANY(tokens_mentioned)"
+                params.append(token)
+            if sort == "sentiment":
+                query += " ORDER BY sentiment_score ASC NULLS LAST"
+            else:
+                query += " ORDER BY published_at DESC"
+            query += f" LIMIT {limit} OFFSET {offset}"
+            async with catalog._pg_pool.acquire() as conn:
+                rows = await conn.fetch(query, *params)
+            for r in rows:
+                items.append(
+                    NewsItemOut(
+                        news_id=r["news_id"],
+                        url=r["url"],
+                        title=r["title"],
+                        summary=r["summary"] or "",
+                        source=r["source"],
+                        published_at=r["published_at"],
+                        chains_mentioned=list(r["chains_mentioned"] or []),
+                        tokens_mentioned=list(r["tokens_mentioned"] or []),
+                        sentiment_score=r["sentiment_score"],
+                    )
+                )
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning(f"news_list_new_fail: {e}")
+
+    # Legacy fallback (crypto_news)
+    if not items and catalog._health.postgres:
+        try:
+            query = "SELECT id, title, content, url, source, sentiment, tickers, published, ingested_at FROM crypto_news WHERE 1=1"
+            params = []
+            if category:
+                query += f" AND category = ${len(params)+1}"
+                params.append(category)
+            query += " ORDER BY ingested_at DESC LIMIT $%d OFFSET $%d" % (len(params)+1, len(params)+2)
+            params.extend([limit, offset])
+            async with catalog._pg_pool.acquire() as conn:
+                rows = await conn.fetch(query, *params)
+            for r in rows:
+                items.append(_adapt_legacy_row(dict(r)))
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning(f"news_list_legacy_fail: {e}")
+
+    # T03: cluster into stories if requested
+    if clustered and items:
+        from app.domain.news.clusterer import NewsItem, cluster_items, persist_clusters
+        cluster_items_list = [
+            NewsItem(
+                id=it.news_id,
+                title=it.title,
+                body=it.summary or "",
+                source=it.source or "",
+                url=it.url or "",
+                published_at=it.published_at,
+                sentiment=it.sentiment_score or 0.0,
+            )
+            for it in items
+        ]
+        stories = cluster_items(cluster_items_list)
+        # persist in background (don't block response)
+        try:
+            import asyncio
+            asyncio.create_task(persist_clusters(stories))
+        except Exception:
+            pass
+        # Return clusters as synthetic items (representative title, first source)
+        clustered_items = []
+        for s in stories:
+            clustered_items.append(
+                NewsItemOut(
+                    news_id=s.cluster_id,
+                    url=s.source_urls[0] if s.source_urls else "",
+                    title=f"[×{s.item_count}] {s.representative_title}",
+                    summary=f"Story across {len(s.sources)} sources. "
+                            f"Sentiment: {s.sentiment_avg:.2f}. "
+                            f"Item IDs: {','.join(s.item_ids[:5])}",
+                    source=", ".join(s.sources[:3]),
+                    published_at=s.last_updated,
+                    chains_mentioned=[],
+                    tokens_mentioned=[],
+                    sentiment_score=s.sentiment_avg,
+                )
+            )
+        return NewsListResponse(items=clustered_items, total=len(clustered_items), offset=offset)
+
+    return NewsListResponse(items=items, total=len(items), offset=offset)
+
+
+# ── GET /api/v1/news/trending ─────────────────────────────────────
+@router.get("/trending", response_model=NewsListResponse)
+async def trending_news(
+    window_hours: int = Query(168, ge=1, le=720),
+    limit: int = Query(20, ge=1, le=100),
+) -> NewsListResponse:
+    """Time-decay trending. Reads from news_items (new) primary, falls back to crypto_news (legacy)."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    if not catalog._health.postgres:
+        return NewsListResponse(items=[], total=0, offset=0)
+    now = utcnow()
+    cutoff = now - timedelta(hours=window_hours)
+    items: list[NewsItemOut] = []
+    # Primary: news_items
+    try:
+        async with catalog._pg_pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT news_id, url, title, summary, source, published_at, "
+                "sentiment_score, chains_mentioned, tokens_mentioned "
+                "FROM news_items "
+                "WHERE published_at > $1 "
+                "ORDER BY published_at DESC LIMIT 500",
+                cutoff,
+            )
+        for r in rows:
+            hours_old = max(0, (now - r["published_at"]).total_seconds() / 3600)
+            item = NewsItemOut(
+                news_id=r["news_id"],
+                url=r["url"] or "",
+                title=r["title"] or "",
+                summary=r["summary"] or "",
+                source=r["source"] or "unknown",
+                published_at=r["published_at"],
+                chains_mentioned=list(r["chains_mentioned"] or []),
+                tokens_mentioned=list(r["tokens_mentioned"] or []),
+                sentiment_score=r["sentiment_score"],
+            )
+            item.score = round(
+                trend_score(hours_old, item.source, item.sentiment_score), 4
+            )
+            items.append(item)
+    except Exception as e:
+        import logging
+        logging.getLogger(__name__).warning(f"trending_new_fail: {e}")
+
+    # Fallback: crypto_news (legacy) if no new items
+    if not items:
+        try:
+            cutoff_epoch = cutoff.timestamp()
+            async with catalog._pg_pool.acquire() as conn:
+                rows = await conn.fetch(
+                    "SELECT id, title, content, url, source, sentiment, tickers, "
+                    "published, ingested_at, category "
+                    "FROM crypto_news "
+                    "WHERE ingested_at > $1 "
+                    "ORDER BY ingested_at DESC LIMIT 500",
+                    cutoff_epoch,
+                )
+            for r in rows:
+                d = dict(r)
+                hours_old = 0.0
+                try:
+                    if d.get("ingested_at"):
+                        hours_old = max(0, (now.timestamp() - float(d["ingested_at"])) / 3600)
+                except Exception:
+                    pass
+                item = _adapt_legacy_row(d)
+                item.score = round(trend_score(hours_old, item.source, item.sentiment_score), 4)
+                items.append(item)
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning(f"trending_legacy_fail: {e}")
+
+    items.sort(key=lambda x: x.score or 0, reverse=True)
+    return NewsListResponse(items=items[:limit], total=len(items), offset=0)
+
+
+# ── GET /api/v1/news/{news_id} ────────────────────────────────────
+@router.get("/{news_id}", response_model=NewsItemOut)
+async def get_news(news_id: str) -> NewsItemOut:
+    """Single news item. Searches both news_items and crypto_news."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    if not catalog._health.postgres:
+        raise HTTPException(503, "postgres unavailable")
+    try:
+        async with catalog._pg_pool.acquire() as conn:
+            r = await conn.fetchrow(
+                "SELECT news_id, url, title, summary, source, published_at, "
+                "sentiment_score, chains_mentioned, tokens_mentioned "
+                "FROM news_items WHERE news_id=$1",
+                news_id,
+            )
+            if r:
+                return NewsItemOut(
+                    news_id=r["news_id"],
+                    url=r["url"],
+                    title=r["title"],
+                    summary=r["summary"] or "",
+                    source=r["source"],
+                    published_at=r["published_at"],
+                    chains_mentioned=list(r["chains_mentioned"] or []),
+                    tokens_mentioned=list(r["tokens_mentioned"] or []),
+                    sentiment_score=r["sentiment_score"],
+                )
+            r2 = await conn.fetchrow(
+                "SELECT id, title, content, url, source, sentiment, tickers, "
+                "published, ingested_at, category "
+                "FROM crypto_news WHERE id=$1",
+                news_id,
+            )
+        if r2:
+            return _adapt_legacy_row(dict(r2))
+        raise HTTPException(404, "news item not found")
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(500, f"news_get_fail: {e}")
+
+
+# ── POST /api/v1/news/{news_id}/analyze ───────────────────────────
+@router.post("/{news_id}/analyze", response_model=NewsAnalysisResponse)
+async def analyze_news(news_id: str) -> NewsAnalysisResponse:
+    """Generate LLM analysis via LiteLLM. Falls back to None if LLM unreachable."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    if not catalog._health.postgres:
+        raise HTTPException(503, "postgres unavailable")
+    try:
+        # Fetch the news item
+        item: NewsItemOut | None = None
+        async with catalog._pg_pool.acquire() as conn:
+            r = await conn.fetchrow(
+                "SELECT news_id, url, title, summary, source, published_at, "
+                "sentiment_score, chains_mentioned, tokens_mentioned "
+                "FROM news_items WHERE news_id=$1",
+                news_id,
+            )
+            if r:
+                item = NewsItemOut(
+                    news_id=r["news_id"],
+                    url=r["url"],
+                    title=r["title"],
+                    summary=r["summary"] or "",
+                    source=r["source"],
+                    published_at=r["published_at"],
+                    chains_mentioned=list(r["chains_mentioned"] or []),
+                    tokens_mentioned=list(r["tokens_mentioned"] or []),
+                    sentiment_score=r["sentiment_score"],
+                )
+            if not item:
+                r2 = await conn.fetchrow(
+                    "SELECT id, title, content, url, source, sentiment, tickers, "
+                    "published, ingested_at FROM crypto_news WHERE id=$1",
+                    news_id,
+                )
+                if r2:
+                    item = _adapt_legacy_row(dict(r2))
+        if not item:
+            raise HTTPException(404, "news item not found")
+        # Build a NewsItem for the LLM router
+        from app.catalog.models import NewsItem
+
+        ni = NewsItem(
+            news_id=item.news_id,
+            url=item.url or "https://unknown.local",  # HttpUrl requires non-empty
+            title=item.title or "",
+            summary=item.summary or "",
+            body_markdown=item.summary or "",
+            source=item.source or "unknown",
+            published_at=item.published_at or utcnow(),
+            ingested_at=utcnow(),
+        )
+        llm = LLMRouter()
+        analysis = await llm.analyze_news(ni)
+        if analysis is None:
+            return NewsAnalysisResponse(
+                news_id=news_id, analysis=None, error="LLM router unavailable"
+            )
+        return NewsAnalysisResponse(
+            news_id=news_id, analysis=analysis, model="deepseek-v3"
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        return NewsAnalysisResponse(news_id=news_id, analysis=None, error=str(e))