merge: chore/cleanup-remove-bloat-and-secrets into main

app/domain/news/router.py

@@ -0,0 +1,450 @@
+"""T28 News Intelligence Product.
+
+Per v4.0 §T28. Four endpoints surface the news pipeline as a product:
+  POST /api/v1/news              list, paginated, filterable
+  GET  /api/v1/news/trending     time-decay weighted
+  GET  /api/v1/news/{news_id}    single item
+  POST /api/v1/news/{news_id}/analyze  LLM analysis (LiteLLM)
+
+Data sources:
+  - crypto_news (legacy table, 1750+ items from RSS feeds)
+  - news_items (new catalog table, populated by RSS ingest)
+  - Qdrant embeddings for semantic search (rag_embedding_id)
+
+Time-decay scoring for /trending:
+  score = recency_decay * source_authority * social_velocity * abs(sentiment)
+  recency_decay = 0.5 ** (hours_old / 6)        half-life 6h
+  source_authority: tier-1 (CoinDesk, The Block) = 1.0, tier-2 = 0.5, tier-3 = 0.2
+  social_velocity: tweets/shares in last hour (1 + n/100, capped at 2x)
+  abs(sentiment): polarizing news ranks higher
+"""
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.catalog.llm_router import LLMRouter
+from app.catalog.models import utcnow
+from app.catalog.service import get_catalog
+
+router = APIRouter(prefix="/api/v1/news", tags=["news"])
+
+
+# ── Time-decay scoring (per v4.0 §T28) ────────────────────────────
+SOURCE_AUTHORITY: dict[str, float] = {
+    # Tier 1 — major crypto-native outlets
+    "coindesk": 1.0,
+    "the block": 1.0,
+    "decrypt": 1.0,
+    "cointelegraph": 1.0,
+    # Tier 2 — solid crypto coverage
+    "beincrypto": 0.7,
+    "u.today": 0.7,
+    "crypto.news": 0.7,
+    "blockworks": 0.8,
+    # Tier 3 — general / RSS aggregators
+    "google-crypto": 0.5,
+    "reddit-crypto": 0.4,
+    "twitter-crypto": 0.3,
+}
+
+
+def recency_decay(hours_old: float, half_life: float = 6.0) -> float:
+    """Exponential decay: 1.0 at 0h, 0.5 at half_life, 0.25 at 2*half_life."""
+    if hours_old < 0:
+        return 1.0
+    return 0.5 ** (hours_old / half_life)
+
+
+def source_authority(source: str) -> float:
+    s = (source or "").lower().strip()
+    for key, val in SOURCE_AUTHORITY.items():
+        if key in s:
+            return val
+    return 0.3  # unknown source
+
+
+def trend_score(
+    hours_old: float, source: str, sentiment: float | None, social_velocity: int = 0
+) -> float:
+    """Composite trending score per v4.0 formula."""
+    s_auth = source_authority(source)
+    s_vel = min(2.0, 1.0 + social_velocity / 100.0)
+    s_sent = 1.0 + abs(sentiment or 0.0)
+    return recency_decay(hours_old) * s_auth * s_vel * s_sent
+
+
+# ── Response models ────────────────────────────────────────────────
+class NewsItemOut(BaseModel):
+    model_config = ConfigDict(strict=False)  # accept None for any field with default
+
+    news_id: str | None = None
+    url: str | None = None
+    title: str | None = None
+    summary: str | None = ""
+    source: str | None = "unknown"
+    published_at: datetime | None = None
+    chains_mentioned: list[str] = Field(default_factory=list)
+    tokens_mentioned: list[str] = Field(default_factory=list)
+    sentiment_score: float | None = None
+    score: float | None = None  # only set for /trending
+
+
+class NewsListResponse(BaseModel):
+    items: list[NewsItemOut]
+    total: int
+    offset: int
+
+
+class NewsAnalysisResponse(BaseModel):
+    news_id: str
+    analysis: str | None
+    model: str | None = None
+    error: str | None = None
+
+
+# ── Row adapter (legacy crypto_news → NewsItemOut) ───────────────
+def _adapt_legacy_row(row: dict) -> NewsItemOut:
+    """Convert a crypto_news row to NewsItemOut shape."""
+    # published is a text field in legacy; try ISO parse
+    pub_str = row.get("published") or row.get("ingested_at")
+    pub_dt = utcnow()
+    if pub_str:
+        try:
+            if isinstance(pub_str, (int, float)):
+                pub_dt = datetime.fromtimestamp(float(pub_str), tz=UTC)
+            else:
+                # Try common formats
+                for fmt in (
+                    "%Y-%m-%dT%H:%M:%S.%fZ",
+                    "%Y-%m-%dT%H:%M:%SZ",
+                    "%Y-%m-%dT%H:%M:%S",
+                    "%Y-%m-%d %H:%M:%S",
+                ):
+                    try:
+                        pub_dt = datetime.strptime(str(pub_str)[:19], fmt).replace(tzinfo=UTC)
+                        break
+                    except ValueError:
+                        continue
+        except Exception:
+            pass
+    return NewsItemOut(
+        news_id=row.get("id", ""),
+        url=row.get("url", ""),
+        title=row.get("title", ""),
+        summary=(row.get("content") or "")[:500],
+        source=row.get("source", "unknown"),
+        published_at=pub_dt,
+        chains_mentioned=[],
+        tokens_mentioned=row.get("tickers") or [],
+        sentiment_score=row.get("sentiment"),
+    )
+
+
+# ── POST /api/v1/news (list with filters) ─────────────────────────
+@router.post("", response_model=NewsListResponse)
+async def list_news(
+    chain: str | None = None,
+    token: str | None = None,
+    category: str | None = None,
+    since_hours: int = Query(24, ge=1, le=720),
+    limit: int = Query(20, ge=1, le=200),
+    offset: int = Query(0, ge=0),
+    sort: str = Query("recency", pattern="^(recency|relevance|sentiment)$"),
+    clustered: bool = Query(False, description="T03: dedupe via MinHash+DBSCAN into stories"),
+) -> NewsListResponse:
+    """List news items with filters. Reads from both news_items (new) and crypto_news (legacy)."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    items: list[NewsItemOut] = []
+
+    # New table
+    if catalog._health.postgres:
+        try:
+            cutoff = utcnow() - timedelta(hours=since_hours)
+            query = "SELECT news_id, url, title, summary, source, published_at, sentiment_score, chains_mentioned, tokens_mentioned FROM news_items WHERE published_at > $1"
+            params: list[Any] = [cutoff]
+            if chain:
+                query += f" AND ${len(params)+1} = ANY(chains_mentioned)"
+                params.append(chain)
+            if token:
+                query += f" AND ${len(params)+1} = ANY(tokens_mentioned)"
+                params.append(token)
+            if sort == "sentiment":
+                query += " ORDER BY sentiment_score ASC NULLS LAST"
+            else:
+                query += " ORDER BY published_at DESC"
+            query += f" LIMIT {limit} OFFSET {offset}"
+            async with catalog._pg_pool.acquire() as conn:
+                rows = await conn.fetch(query, *params)
+            for r in rows:
+                items.append(
+                    NewsItemOut(
+                        news_id=r["news_id"],
+                        url=r["url"],
+                        title=r["title"],
+                        summary=r["summary"] or "",
+                        source=r["source"],
+                        published_at=r["published_at"],
+                        chains_mentioned=list(r["chains_mentioned"] or []),
+                        tokens_mentioned=list(r["tokens_mentioned"] or []),
+                        sentiment_score=r["sentiment_score"],
+                    )
+                )
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning(f"news_list_new_fail: {e}")
+
+    # Legacy fallback (crypto_news)
+    if not items and catalog._health.postgres:
+        try:
+            query = "SELECT id, title, content, url, source, sentiment, tickers, published, ingested_at FROM crypto_news WHERE 1=1"
+            params = []
+            if category:
+                query += f" AND category = ${len(params)+1}"
+                params.append(category)
+            query += " ORDER BY ingested_at DESC LIMIT $%d OFFSET $%d" % (len(params)+1, len(params)+2)
+            params.extend([limit, offset])
+            async with catalog._pg_pool.acquire() as conn:
+                rows = await conn.fetch(query, *params)
+            for r in rows:
+                items.append(_adapt_legacy_row(dict(r)))
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning(f"news_list_legacy_fail: {e}")
+
+    # T03: cluster into stories if requested
+    if clustered and items:
+        from app.domain.news.clusterer import NewsItem, cluster_items, persist_clusters
+        cluster_items_list = [
+            NewsItem(
+                id=it.news_id,
+                title=it.title,
+                body=it.summary or "",
+                source=it.source or "",
+                url=it.url or "",
+                published_at=it.published_at,
+                sentiment=it.sentiment_score or 0.0,
+            )
+            for it in items
+        ]
+        stories = cluster_items(cluster_items_list)
+        # persist in background (don't block response)
+        try:
+            import asyncio
+            asyncio.create_task(persist_clusters(stories))
+        except Exception:
+            pass
+        # Return clusters as synthetic items (representative title, first source)
+        clustered_items = []
+        for s in stories:
+            clustered_items.append(
+                NewsItemOut(
+                    news_id=s.cluster_id,
+                    url=s.source_urls[0] if s.source_urls else "",
+                    title=f"[×{s.item_count}] {s.representative_title}",
+                    summary=f"Story across {len(s.sources)} sources. "
+                            f"Sentiment: {s.sentiment_avg:.2f}. "
+                            f"Item IDs: {','.join(s.item_ids[:5])}",
+                    source=", ".join(s.sources[:3]),
+                    published_at=s.last_updated,
+                    chains_mentioned=[],
+                    tokens_mentioned=[],
+                    sentiment_score=s.sentiment_avg,
+                )
+            )
+        return NewsListResponse(items=clustered_items, total=len(clustered_items), offset=offset)
+
+    return NewsListResponse(items=items, total=len(items), offset=offset)
+
+
+# ── GET /api/v1/news/trending ─────────────────────────────────────
+@router.get("/trending", response_model=NewsListResponse)
+async def trending_news(
+    window_hours: int = Query(168, ge=1, le=720),
+    limit: int = Query(20, ge=1, le=100),
+) -> NewsListResponse:
+    """Time-decay trending. Reads from news_items (new) primary, falls back to crypto_news (legacy)."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    if not catalog._health.postgres:
+        return NewsListResponse(items=[], total=0, offset=0)
+    now = utcnow()
+    cutoff = now - timedelta(hours=window_hours)
+    items: list[NewsItemOut] = []
+    # Primary: news_items
+    try:
+        async with catalog._pg_pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT news_id, url, title, summary, source, published_at, "
+                "sentiment_score, chains_mentioned, tokens_mentioned "
+                "FROM news_items "
+                "WHERE published_at > $1 "
+                "ORDER BY published_at DESC LIMIT 500",
+                cutoff,
+            )
+        for r in rows:
+            hours_old = max(0, (now - r["published_at"]).total_seconds() / 3600)
+            item = NewsItemOut(
+                news_id=r["news_id"],
+                url=r["url"] or "",
+                title=r["title"] or "",
+                summary=r["summary"] or "",
+                source=r["source"] or "unknown",
+                published_at=r["published_at"],
+                chains_mentioned=list(r["chains_mentioned"] or []),
+                tokens_mentioned=list(r["tokens_mentioned"] or []),
+                sentiment_score=r["sentiment_score"],
+            )
+            item.score = round(
+                trend_score(hours_old, item.source, item.sentiment_score), 4
+            )
+            items.append(item)
+    except Exception as e:
+        import logging
+        logging.getLogger(__name__).warning(f"trending_new_fail: {e}")
+
+    # Fallback: crypto_news (legacy) if no new items
+    if not items:
+        try:
+            cutoff_epoch = cutoff.timestamp()
+            async with catalog._pg_pool.acquire() as conn:
+                rows = await conn.fetch(
+                    "SELECT id, title, content, url, source, sentiment, tickers, "
+                    "published, ingested_at, category "
+                    "FROM crypto_news "
+                    "WHERE ingested_at > $1 "
+                    "ORDER BY ingested_at DESC LIMIT 500",
+                    cutoff_epoch,
+                )
+            for r in rows:
+                d = dict(r)
+                hours_old = 0.0
+                try:
+                    if d.get("ingested_at"):
+                        hours_old = max(0, (now.timestamp() - float(d["ingested_at"])) / 3600)
+                except Exception:
+                    pass
+                item = _adapt_legacy_row(d)
+                item.score = round(trend_score(hours_old, item.source, item.sentiment_score), 4)
+                items.append(item)
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning(f"trending_legacy_fail: {e}")
+
+    items.sort(key=lambda x: x.score or 0, reverse=True)
+    return NewsListResponse(items=items[:limit], total=len(items), offset=0)
+
+
+# ── GET /api/v1/news/{news_id} ────────────────────────────────────
+@router.get("/{news_id}", response_model=NewsItemOut)
+async def get_news(news_id: str) -> NewsItemOut:
+    """Single news item. Searches both news_items and crypto_news."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    if not catalog._health.postgres:
+        raise HTTPException(503, "postgres unavailable")
+    try:
+        async with catalog._pg_pool.acquire() as conn:
+            r = await conn.fetchrow(
+                "SELECT news_id, url, title, summary, source, published_at, "
+                "sentiment_score, chains_mentioned, tokens_mentioned "
+                "FROM news_items WHERE news_id=$1",
+                news_id,
+            )
+            if r:
+                return NewsItemOut(
+                    news_id=r["news_id"],
+                    url=r["url"],
+                    title=r["title"],
+                    summary=r["summary"] or "",
+                    source=r["source"],
+                    published_at=r["published_at"],
+                    chains_mentioned=list(r["chains_mentioned"] or []),
+                    tokens_mentioned=list(r["tokens_mentioned"] or []),
+                    sentiment_score=r["sentiment_score"],
+                )
+            r2 = await conn.fetchrow(
+                "SELECT id, title, content, url, source, sentiment, tickers, "
+                "published, ingested_at, category "
+                "FROM crypto_news WHERE id=$1",
+                news_id,
+            )
+        if r2:
+            return _adapt_legacy_row(dict(r2))
+        raise HTTPException(404, "news item not found")
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(500, f"news_get_fail: {e}")
+
+
+# ── POST /api/v1/news/{news_id}/analyze ───────────────────────────
+@router.post("/{news_id}/analyze", response_model=NewsAnalysisResponse)
+async def analyze_news(news_id: str) -> NewsAnalysisResponse:
+    """Generate LLM analysis via LiteLLM. Falls back to None if LLM unreachable."""
+    catalog = get_catalog()
+    await catalog._init_stores()
+    if not catalog._health.postgres:
+        raise HTTPException(503, "postgres unavailable")
+    try:
+        # Fetch the news item
+        item: NewsItemOut | None = None
+        async with catalog._pg_pool.acquire() as conn:
+            r = await conn.fetchrow(
+                "SELECT news_id, url, title, summary, source, published_at, "
+                "sentiment_score, chains_mentioned, tokens_mentioned "
+                "FROM news_items WHERE news_id=$1",
+                news_id,
+            )
+            if r:
+                item = NewsItemOut(
+                    news_id=r["news_id"],
+                    url=r["url"],
+                    title=r["title"],
+                    summary=r["summary"] or "",
+                    source=r["source"],
+                    published_at=r["published_at"],
+                    chains_mentioned=list(r["chains_mentioned"] or []),
+                    tokens_mentioned=list(r["tokens_mentioned"] or []),
+                    sentiment_score=r["sentiment_score"],
+                )
+            if not item:
+                r2 = await conn.fetchrow(
+                    "SELECT id, title, content, url, source, sentiment, tickers, "
+                    "published, ingested_at FROM crypto_news WHERE id=$1",
+                    news_id,
+                )
+                if r2:
+                    item = _adapt_legacy_row(dict(r2))
+        if not item:
+            raise HTTPException(404, "news item not found")
+        # Build a NewsItem for the LLM router
+        from app.catalog.models import NewsItem
+
+        ni = NewsItem(
+            news_id=item.news_id,
+            url=item.url or "https://unknown.local",  # HttpUrl requires non-empty
+            title=item.title or "",
+            summary=item.summary or "",
+            body_markdown=item.summary or "",
+            source=item.source or "unknown",
+            published_at=item.published_at or utcnow(),
+            ingested_at=utcnow(),
+        )
+        llm = LLMRouter()
+        analysis = await llm.analyze_news(ni)
+        if analysis is None:
+            return NewsAnalysisResponse(
+                news_id=news_id, analysis=None, error="LLM router unavailable"
+            )
+        return NewsAnalysisResponse(
+            news_id=news_id, analysis=analysis, model="deepseek-v3"
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        return NewsAnalysisResponse(news_id=news_id, analysis=None, error=str(e))