450 lines
18 KiB
Python
450 lines
18 KiB
Python
"""T28 News Intelligence Product.
|
||
|
||
Per v4.0 §T28. Four endpoints surface the news pipeline as a product:
|
||
POST /api/v1/news list, paginated, filterable
|
||
GET /api/v1/news/trending time-decay weighted
|
||
GET /api/v1/news/{news_id} single item
|
||
POST /api/v1/news/{news_id}/analyze LLM analysis (LiteLLM)
|
||
|
||
Data sources:
|
||
- crypto_news (legacy table, 1750+ items from RSS feeds)
|
||
- news_items (new catalog table, populated by RSS ingest)
|
||
- Qdrant embeddings for semantic search (rag_embedding_id)
|
||
|
||
Time-decay scoring for /trending:
|
||
score = recency_decay * source_authority * social_velocity * abs(sentiment)
|
||
recency_decay = 0.5 ** (hours_old / 6) half-life 6h
|
||
source_authority: tier-1 (CoinDesk, The Block) = 1.0, tier-2 = 0.5, tier-3 = 0.2
|
||
social_velocity: tweets/shares in last hour (1 + n/100, capped at 2x)
|
||
abs(sentiment): polarizing news ranks higher
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
|
||
from fastapi import APIRouter, HTTPException, Query
|
||
from pydantic import BaseModel, ConfigDict, Field
|
||
|
||
from app.catalog.llm_router import LLMRouter
|
||
from app.catalog.models import utcnow
|
||
from app.catalog.service import get_catalog
|
||
|
||
router = APIRouter(prefix="/api/v1/news", tags=["news"])
|
||
|
||
|
||
# ── Time-decay scoring (per v4.0 §T28) ────────────────────────────
|
||
SOURCE_AUTHORITY: dict[str, float] = {
|
||
# Tier 1 — major crypto-native outlets
|
||
"coindesk": 1.0,
|
||
"the block": 1.0,
|
||
"decrypt": 1.0,
|
||
"cointelegraph": 1.0,
|
||
# Tier 2 — solid crypto coverage
|
||
"beincrypto": 0.7,
|
||
"u.today": 0.7,
|
||
"crypto.news": 0.7,
|
||
"blockworks": 0.8,
|
||
# Tier 3 — general / RSS aggregators
|
||
"google-crypto": 0.5,
|
||
"reddit-crypto": 0.4,
|
||
"twitter-crypto": 0.3,
|
||
}
|
||
|
||
|
||
def recency_decay(hours_old: float, half_life: float = 6.0) -> float:
|
||
"""Exponential decay: 1.0 at 0h, 0.5 at half_life, 0.25 at 2*half_life."""
|
||
if hours_old < 0:
|
||
return 1.0
|
||
return 0.5 ** (hours_old / half_life)
|
||
|
||
|
||
def source_authority(source: str) -> float:
|
||
s = (source or "").lower().strip()
|
||
for key, val in SOURCE_AUTHORITY.items():
|
||
if key in s:
|
||
return val
|
||
return 0.3 # unknown source
|
||
|
||
|
||
def trend_score(
|
||
hours_old: float, source: str, sentiment: float | None, social_velocity: int = 0
|
||
) -> float:
|
||
"""Composite trending score per v4.0 formula."""
|
||
s_auth = source_authority(source)
|
||
s_vel = min(2.0, 1.0 + social_velocity / 100.0)
|
||
s_sent = 1.0 + abs(sentiment or 0.0)
|
||
return recency_decay(hours_old) * s_auth * s_vel * s_sent
|
||
|
||
|
||
# ── Response models ────────────────────────────────────────────────
|
||
class NewsItemOut(BaseModel):
|
||
model_config = ConfigDict(strict=False) # accept None for any field with default
|
||
|
||
news_id: str | None = None
|
||
url: str | None = None
|
||
title: str | None = None
|
||
summary: str | None = ""
|
||
source: str | None = "unknown"
|
||
published_at: datetime | None = None
|
||
chains_mentioned: list[str] = Field(default_factory=list)
|
||
tokens_mentioned: list[str] = Field(default_factory=list)
|
||
sentiment_score: float | None = None
|
||
score: float | None = None # only set for /trending
|
||
|
||
|
||
class NewsListResponse(BaseModel):
|
||
items: list[NewsItemOut]
|
||
total: int
|
||
offset: int
|
||
|
||
|
||
class NewsAnalysisResponse(BaseModel):
|
||
news_id: str
|
||
analysis: str | None
|
||
model: str | None = None
|
||
error: str | None = None
|
||
|
||
|
||
# ── Row adapter (legacy crypto_news → NewsItemOut) ───────────────
|
||
def _adapt_legacy_row(row: dict) -> NewsItemOut:
|
||
"""Convert a crypto_news row to NewsItemOut shape."""
|
||
# published is a text field in legacy; try ISO parse
|
||
pub_str = row.get("published") or row.get("ingested_at")
|
||
pub_dt = utcnow()
|
||
if pub_str:
|
||
try:
|
||
if isinstance(pub_str, (int, float)):
|
||
pub_dt = datetime.fromtimestamp(float(pub_str), tz=UTC)
|
||
else:
|
||
# Try common formats
|
||
for fmt in (
|
||
"%Y-%m-%dT%H:%M:%S.%fZ",
|
||
"%Y-%m-%dT%H:%M:%SZ",
|
||
"%Y-%m-%dT%H:%M:%S",
|
||
"%Y-%m-%d %H:%M:%S",
|
||
):
|
||
try:
|
||
pub_dt = datetime.strptime(str(pub_str)[:19], fmt).replace(tzinfo=UTC)
|
||
break
|
||
except ValueError:
|
||
continue
|
||
except Exception:
|
||
pass
|
||
return NewsItemOut(
|
||
news_id=row.get("id", ""),
|
||
url=row.get("url", ""),
|
||
title=row.get("title", ""),
|
||
summary=(row.get("content") or "")[:500],
|
||
source=row.get("source", "unknown"),
|
||
published_at=pub_dt,
|
||
chains_mentioned=[],
|
||
tokens_mentioned=row.get("tickers") or [],
|
||
sentiment_score=row.get("sentiment"),
|
||
)
|
||
|
||
|
||
# ── POST /api/v1/news (list with filters) ─────────────────────────
|
||
@router.post("", response_model=NewsListResponse)
|
||
async def list_news(
|
||
chain: str | None = None,
|
||
token: str | None = None,
|
||
category: str | None = None,
|
||
since_hours: int = Query(24, ge=1, le=720),
|
||
limit: int = Query(20, ge=1, le=200),
|
||
offset: int = Query(0, ge=0),
|
||
sort: str = Query("recency", pattern="^(recency|relevance|sentiment)$"),
|
||
clustered: bool = Query(False, description="T03: dedupe via MinHash+DBSCAN into stories"),
|
||
) -> NewsListResponse:
|
||
"""List news items with filters. Reads from both news_items (new) and crypto_news (legacy)."""
|
||
catalog = get_catalog()
|
||
await catalog._init_stores()
|
||
items: list[NewsItemOut] = []
|
||
|
||
# New table
|
||
if catalog._health.postgres:
|
||
try:
|
||
cutoff = utcnow() - timedelta(hours=since_hours)
|
||
query = "SELECT news_id, url, title, summary, source, published_at, sentiment_score, chains_mentioned, tokens_mentioned FROM news_items WHERE published_at > $1"
|
||
params: list[Any] = [cutoff]
|
||
if chain:
|
||
query += f" AND ${len(params)+1} = ANY(chains_mentioned)"
|
||
params.append(chain)
|
||
if token:
|
||
query += f" AND ${len(params)+1} = ANY(tokens_mentioned)"
|
||
params.append(token)
|
||
if sort == "sentiment":
|
||
query += " ORDER BY sentiment_score ASC NULLS LAST"
|
||
else:
|
||
query += " ORDER BY published_at DESC"
|
||
query += f" LIMIT {limit} OFFSET {offset}"
|
||
async with catalog._pg_pool.acquire() as conn:
|
||
rows = await conn.fetch(query, *params)
|
||
for r in rows:
|
||
items.append(
|
||
NewsItemOut(
|
||
news_id=r["news_id"],
|
||
url=r["url"],
|
||
title=r["title"],
|
||
summary=r["summary"] or "",
|
||
source=r["source"],
|
||
published_at=r["published_at"],
|
||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||
sentiment_score=r["sentiment_score"],
|
||
)
|
||
)
|
||
except Exception as e:
|
||
import logging
|
||
logging.getLogger(__name__).warning(f"news_list_new_fail: {e}")
|
||
|
||
# Legacy fallback (crypto_news)
|
||
if not items and catalog._health.postgres:
|
||
try:
|
||
query = "SELECT id, title, content, url, source, sentiment, tickers, published, ingested_at FROM crypto_news WHERE 1=1"
|
||
params = []
|
||
if category:
|
||
query += f" AND category = ${len(params)+1}"
|
||
params.append(category)
|
||
query += " ORDER BY ingested_at DESC LIMIT $%d OFFSET $%d" % (len(params)+1, len(params)+2)
|
||
params.extend([limit, offset])
|
||
async with catalog._pg_pool.acquire() as conn:
|
||
rows = await conn.fetch(query, *params)
|
||
for r in rows:
|
||
items.append(_adapt_legacy_row(dict(r)))
|
||
except Exception as e:
|
||
import logging
|
||
logging.getLogger(__name__).warning(f"news_list_legacy_fail: {e}")
|
||
|
||
# T03: cluster into stories if requested
|
||
if clustered and items:
|
||
from app.domain.news.clusterer import NewsItem, cluster_items, persist_clusters
|
||
cluster_items_list = [
|
||
NewsItem(
|
||
id=it.news_id,
|
||
title=it.title,
|
||
body=it.summary or "",
|
||
source=it.source or "",
|
||
url=it.url or "",
|
||
published_at=it.published_at,
|
||
sentiment=it.sentiment_score or 0.0,
|
||
)
|
||
for it in items
|
||
]
|
||
stories = cluster_items(cluster_items_list)
|
||
# persist in background (don't block response)
|
||
try:
|
||
import asyncio
|
||
asyncio.create_task(persist_clusters(stories))
|
||
except Exception:
|
||
pass
|
||
# Return clusters as synthetic items (representative title, first source)
|
||
clustered_items = []
|
||
for s in stories:
|
||
clustered_items.append(
|
||
NewsItemOut(
|
||
news_id=s.cluster_id,
|
||
url=s.source_urls[0] if s.source_urls else "",
|
||
title=f"[×{s.item_count}] {s.representative_title}",
|
||
summary=f"Story across {len(s.sources)} sources. "
|
||
f"Sentiment: {s.sentiment_avg:.2f}. "
|
||
f"Item IDs: {','.join(s.item_ids[:5])}",
|
||
source=", ".join(s.sources[:3]),
|
||
published_at=s.last_updated,
|
||
chains_mentioned=[],
|
||
tokens_mentioned=[],
|
||
sentiment_score=s.sentiment_avg,
|
||
)
|
||
)
|
||
return NewsListResponse(items=clustered_items, total=len(clustered_items), offset=offset)
|
||
|
||
return NewsListResponse(items=items, total=len(items), offset=offset)
|
||
|
||
|
||
# ── GET /api/v1/news/trending ─────────────────────────────────────
|
||
@router.get("/trending", response_model=NewsListResponse)
|
||
async def trending_news(
|
||
window_hours: int = Query(168, ge=1, le=720),
|
||
limit: int = Query(20, ge=1, le=100),
|
||
) -> NewsListResponse:
|
||
"""Time-decay trending. Reads from news_items (new) primary, falls back to crypto_news (legacy)."""
|
||
catalog = get_catalog()
|
||
await catalog._init_stores()
|
||
if not catalog._health.postgres:
|
||
return NewsListResponse(items=[], total=0, offset=0)
|
||
now = utcnow()
|
||
cutoff = now - timedelta(hours=window_hours)
|
||
items: list[NewsItemOut] = []
|
||
# Primary: news_items
|
||
try:
|
||
async with catalog._pg_pool.acquire() as conn:
|
||
rows = await conn.fetch(
|
||
"SELECT news_id, url, title, summary, source, published_at, "
|
||
"sentiment_score, chains_mentioned, tokens_mentioned "
|
||
"FROM news_items "
|
||
"WHERE published_at > $1 "
|
||
"ORDER BY published_at DESC LIMIT 500",
|
||
cutoff,
|
||
)
|
||
for r in rows:
|
||
hours_old = max(0, (now - r["published_at"]).total_seconds() / 3600)
|
||
item = NewsItemOut(
|
||
news_id=r["news_id"],
|
||
url=r["url"] or "",
|
||
title=r["title"] or "",
|
||
summary=r["summary"] or "",
|
||
source=r["source"] or "unknown",
|
||
published_at=r["published_at"],
|
||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||
sentiment_score=r["sentiment_score"],
|
||
)
|
||
item.score = round(
|
||
trend_score(hours_old, item.source, item.sentiment_score), 4
|
||
)
|
||
items.append(item)
|
||
except Exception as e:
|
||
import logging
|
||
logging.getLogger(__name__).warning(f"trending_new_fail: {e}")
|
||
|
||
# Fallback: crypto_news (legacy) if no new items
|
||
if not items:
|
||
try:
|
||
cutoff_epoch = cutoff.timestamp()
|
||
async with catalog._pg_pool.acquire() as conn:
|
||
rows = await conn.fetch(
|
||
"SELECT id, title, content, url, source, sentiment, tickers, "
|
||
"published, ingested_at, category "
|
||
"FROM crypto_news "
|
||
"WHERE ingested_at > $1 "
|
||
"ORDER BY ingested_at DESC LIMIT 500",
|
||
cutoff_epoch,
|
||
)
|
||
for r in rows:
|
||
d = dict(r)
|
||
hours_old = 0.0
|
||
try:
|
||
if d.get("ingested_at"):
|
||
hours_old = max(0, (now.timestamp() - float(d["ingested_at"])) / 3600)
|
||
except Exception:
|
||
pass
|
||
item = _adapt_legacy_row(d)
|
||
item.score = round(trend_score(hours_old, item.source, item.sentiment_score), 4)
|
||
items.append(item)
|
||
except Exception as e:
|
||
import logging
|
||
logging.getLogger(__name__).warning(f"trending_legacy_fail: {e}")
|
||
|
||
items.sort(key=lambda x: x.score or 0, reverse=True)
|
||
return NewsListResponse(items=items[:limit], total=len(items), offset=0)
|
||
|
||
|
||
# ── GET /api/v1/news/{news_id} ────────────────────────────────────
|
||
@router.get("/{news_id}", response_model=NewsItemOut)
|
||
async def get_news(news_id: str) -> NewsItemOut:
|
||
"""Single news item. Searches both news_items and crypto_news."""
|
||
catalog = get_catalog()
|
||
await catalog._init_stores()
|
||
if not catalog._health.postgres:
|
||
raise HTTPException(503, "postgres unavailable")
|
||
try:
|
||
async with catalog._pg_pool.acquire() as conn:
|
||
r = await conn.fetchrow(
|
||
"SELECT news_id, url, title, summary, source, published_at, "
|
||
"sentiment_score, chains_mentioned, tokens_mentioned "
|
||
"FROM news_items WHERE news_id=$1",
|
||
news_id,
|
||
)
|
||
if r:
|
||
return NewsItemOut(
|
||
news_id=r["news_id"],
|
||
url=r["url"],
|
||
title=r["title"],
|
||
summary=r["summary"] or "",
|
||
source=r["source"],
|
||
published_at=r["published_at"],
|
||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||
sentiment_score=r["sentiment_score"],
|
||
)
|
||
r2 = await conn.fetchrow(
|
||
"SELECT id, title, content, url, source, sentiment, tickers, "
|
||
"published, ingested_at, category "
|
||
"FROM crypto_news WHERE id=$1",
|
||
news_id,
|
||
)
|
||
if r2:
|
||
return _adapt_legacy_row(dict(r2))
|
||
raise HTTPException(404, "news item not found")
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
raise HTTPException(500, f"news_get_fail: {e}")
|
||
|
||
|
||
# ── POST /api/v1/news/{news_id}/analyze ───────────────────────────
|
||
@router.post("/{news_id}/analyze", response_model=NewsAnalysisResponse)
|
||
async def analyze_news(news_id: str) -> NewsAnalysisResponse:
|
||
"""Generate LLM analysis via LiteLLM. Falls back to None if LLM unreachable."""
|
||
catalog = get_catalog()
|
||
await catalog._init_stores()
|
||
if not catalog._health.postgres:
|
||
raise HTTPException(503, "postgres unavailable")
|
||
try:
|
||
# Fetch the news item
|
||
item: NewsItemOut | None = None
|
||
async with catalog._pg_pool.acquire() as conn:
|
||
r = await conn.fetchrow(
|
||
"SELECT news_id, url, title, summary, source, published_at, "
|
||
"sentiment_score, chains_mentioned, tokens_mentioned "
|
||
"FROM news_items WHERE news_id=$1",
|
||
news_id,
|
||
)
|
||
if r:
|
||
item = NewsItemOut(
|
||
news_id=r["news_id"],
|
||
url=r["url"],
|
||
title=r["title"],
|
||
summary=r["summary"] or "",
|
||
source=r["source"],
|
||
published_at=r["published_at"],
|
||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||
sentiment_score=r["sentiment_score"],
|
||
)
|
||
if not item:
|
||
r2 = await conn.fetchrow(
|
||
"SELECT id, title, content, url, source, sentiment, tickers, "
|
||
"published, ingested_at FROM crypto_news WHERE id=$1",
|
||
news_id,
|
||
)
|
||
if r2:
|
||
item = _adapt_legacy_row(dict(r2))
|
||
if not item:
|
||
raise HTTPException(404, "news item not found")
|
||
# Build a NewsItem for the LLM router
|
||
from app.catalog.models import NewsItem
|
||
|
||
ni = NewsItem(
|
||
news_id=item.news_id,
|
||
url=item.url or "https://unknown.local", # HttpUrl requires non-empty
|
||
title=item.title or "",
|
||
summary=item.summary or "",
|
||
body_markdown=item.summary or "",
|
||
source=item.source or "unknown",
|
||
published_at=item.published_at or utcnow(),
|
||
ingested_at=utcnow(),
|
||
)
|
||
llm = LLMRouter()
|
||
analysis = await llm.analyze_news(ni)
|
||
if analysis is None:
|
||
return NewsAnalysisResponse(
|
||
news_id=news_id, analysis=None, error="LLM router unavailable"
|
||
)
|
||
return NewsAnalysisResponse(
|
||
news_id=news_id, analysis=analysis, model="deepseek-v3"
|
||
)
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
return NewsAnalysisResponse(news_id=news_id, analysis=None, error=str(e))
|