415 lines
14 KiB
Python
415 lines
14 KiB
Python
"""T03 — News story clustering (G04 FIX).
|
|
|
|
Per MINIMAX_M3_TASKS.md T03. MinHash + DBSCAN dedupes raw RSS items into
|
|
single "stories" so AI agents don't see the same CoinDesk/The Block story
|
|
counted 2-3x in their signal.
|
|
|
|
Algorithm:
|
|
1. MinHash signature (128 permutations) on shingled title+body (first 500 chars)
|
|
2. DBSCAN clusters within 30-minute windows, Jaccard threshold 0.6, eps=0.15
|
|
3. Each cluster = one story with all source URLs, sentiment avg, item count
|
|
4. Persist clusters to Postgres `news_clusters` table; raw items unchanged
|
|
|
|
Endpoints:
|
|
GET /api/v1/news?clustered=true returns stories (clusters), not raw items
|
|
GET /api/v1/news raw items (legacy)
|
|
|
|
This module is pure logic — no I/O at import time. Router/background job
|
|
call `cluster_items(items) -> list[StoryCluster]`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import UTC, datetime
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ── Tokenization ────────────────────────────────────────────────────
|
|
_TOKEN_RE = re.compile(r"[a-z0-9]{3,}", re.IGNORECASE)
|
|
|
|
|
|
def _shingles(text: str, k: int = 3) -> set[str]:
|
|
"""k-shingle set of lowercased alphanumeric tokens. For Jaccard."""
|
|
if not text:
|
|
return set()
|
|
toks = _TOKEN_RE.findall(text.lower())
|
|
if len(toks) < k:
|
|
return set(toks)
|
|
return {" ".join(toks[i : i + k]) for i in range(len(toks) - k + 1)}
|
|
|
|
|
|
# ── MinHash ─────────────────────────────────────────────────────────
|
|
_NUM_PERM = 128
|
|
_MAX_HASH = (1 << 32) - 1
|
|
|
|
|
|
def _minhash_signature(shingles: set[str], seed: int = 42) -> list[int]:
|
|
"""128-permutation MinHash signature of a shingle set.
|
|
|
|
Uses SHA-256 seeded permutations — fast, deterministic, no numpy.
|
|
"""
|
|
if not shingles:
|
|
return [_MAX_HASH] * _NUM_PERM
|
|
sig: list[int] = []
|
|
for i in range(_NUM_PERM):
|
|
m = _MAX_HASH
|
|
for s in shingles:
|
|
h = int.from_bytes(
|
|
hashlib.sha256(f"{i}:{seed}:{s}".encode()).digest()[:4],
|
|
"big",
|
|
)
|
|
if h < m:
|
|
m = h
|
|
sig.append(m)
|
|
return sig
|
|
|
|
|
|
def _jaccard_minhash(a: list[int], b: list[int]) -> float:
|
|
"""Estimate Jaccard similarity from two MinHash signatures."""
|
|
if not a or not b or len(a) != len(b):
|
|
return 0.0
|
|
return sum(1 for x, y in zip(a, b, strict=False) if x == y) / len(a)
|
|
|
|
|
|
# ── DBSCAN (pure-python, no sklearn dep) ────────────────────────────
|
|
def _dbscan(
|
|
signatures: list[list[int]],
|
|
eps: float = 0.4,
|
|
min_samples: int = 2,
|
|
) -> list[int]:
|
|
"""Density-based clustering. Returns cluster id per item (-1 = noise).
|
|
|
|
Similarity is Jaccard (estimated via MinHash). Neighbours are pairs
|
|
with Jaccard distance <= eps (i.e. similarity >= 1 - eps).
|
|
Default eps=0.4 means similarity >= 0.6 (per T03 spec).
|
|
"""
|
|
n = len(signatures)
|
|
labels = [-1] * n
|
|
cluster_id = 0
|
|
for i in range(n):
|
|
if labels[i] != -1:
|
|
continue
|
|
neighbors = [
|
|
j
|
|
for j in range(n)
|
|
if i != j and (1.0 - _jaccard_minhash(signatures[i], signatures[j])) <= eps
|
|
]
|
|
if len(neighbors) < min_samples - 1:
|
|
# not enough neighbours — mark as noise (may become border later)
|
|
continue
|
|
labels[i] = cluster_id
|
|
seed_set = list(neighbors)
|
|
k = 0
|
|
while k < len(seed_set):
|
|
q = seed_set[k]
|
|
if labels[q] == -1:
|
|
labels[q] = cluster_id
|
|
q_neighbors = [
|
|
j
|
|
for j in range(n)
|
|
if j != q
|
|
and (1.0 - _jaccard_minhash(signatures[q], signatures[j])) <= eps
|
|
]
|
|
if len(q_neighbors) >= min_samples - 1:
|
|
seed_set.extend(q_neighbors)
|
|
elif labels[q] is None or labels[q] == -1:
|
|
labels[q] = cluster_id
|
|
k += 1
|
|
cluster_id += 1
|
|
return labels
|
|
|
|
|
|
# ── Domain types ────────────────────────────────────────────────────
|
|
@dataclass
|
|
class NewsItem:
|
|
"""Minimal news item for clustering. Adapts from DB rows or dicts."""
|
|
|
|
id: str
|
|
title: str
|
|
body: str = ""
|
|
source: str = ""
|
|
url: str = ""
|
|
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
|
sentiment: float = 0.0
|
|
|
|
@classmethod
|
|
def from_row(cls, row: dict) -> NewsItem:
|
|
published = row.get("published_at") or row.get("created_at")
|
|
if isinstance(published, str):
|
|
try:
|
|
published = datetime.fromisoformat(published.replace("Z", "+00:00"))
|
|
except (ValueError, AttributeError):
|
|
published = datetime.now(UTC)
|
|
elif not isinstance(published, datetime):
|
|
published = datetime.now(UTC)
|
|
return cls(
|
|
id=str(row.get("id", row.get("news_id", ""))),
|
|
title=row.get("title", "") or "",
|
|
body=(row.get("body") or row.get("summary") or "")[:500],
|
|
source=row.get("source", "") or "",
|
|
url=row.get("url", "") or "",
|
|
published_at=published,
|
|
sentiment=float(row.get("sentiment", 0.0) or 0.0),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class StoryCluster:
|
|
"""One deduplicated story spanning 1+ source items."""
|
|
|
|
cluster_id: str
|
|
representative_title: str
|
|
source_urls: list[str]
|
|
sources: list[str]
|
|
first_seen: datetime
|
|
last_updated: datetime
|
|
item_count: int
|
|
sentiment_avg: float
|
|
item_ids: list[str]
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"cluster_id": self.cluster_id,
|
|
"representative_title": self.representative_title,
|
|
"source_urls": self.source_urls,
|
|
"sources": self.sources,
|
|
"first_seen": self.first_seen.isoformat(),
|
|
"last_updated": self.last_updated.isoformat(),
|
|
"item_count": self.item_count,
|
|
"sentiment_avg": round(self.sentiment_avg, 3),
|
|
"item_ids": self.item_ids,
|
|
}
|
|
|
|
|
|
# ── Main entry point ────────────────────────────────────────────────
|
|
def cluster_items(
|
|
items: list[NewsItem],
|
|
window_minutes: int = 30,
|
|
eps: float = 0.4,
|
|
min_samples: int = 2,
|
|
) -> list[StoryCluster]:
|
|
"""Cluster news items into stories.
|
|
|
|
Items are first grouped by 30-minute time windows, then DBSCAN runs
|
|
on MinHash signatures within each window. Single-item clusters are
|
|
kept (they're "noise" in DBSCAN terms but valid singleton stories).
|
|
|
|
`eps` is the Jaccard DISTANCE threshold (1 - similarity). Per the
|
|
task spec, two items cluster together when Jaccard similarity >= 0.6,
|
|
so distance <= 0.4, so eps=0.4. Tighten for stricter clusters.
|
|
"""
|
|
t0 = time.time()
|
|
if not items:
|
|
return []
|
|
|
|
# Group by time window
|
|
windows: dict[datetime, list[NewsItem]] = {}
|
|
for it in sorted(items, key=lambda x: x.published_at):
|
|
bucket = it.published_at.replace(
|
|
minute=(it.published_at.minute // window_minutes) * window_minutes,
|
|
second=0,
|
|
microsecond=0,
|
|
)
|
|
windows.setdefault(bucket, []).append(it)
|
|
|
|
stories: list[StoryCluster] = []
|
|
for _bucket, group in windows.items():
|
|
if len(group) == 1:
|
|
# singleton — still a story
|
|
it = group[0]
|
|
stories.append(
|
|
StoryCluster(
|
|
cluster_id=hashlib.sha1(
|
|
f"single:{it.id}:{it.published_at.isoformat()}".encode()
|
|
).hexdigest()[:16],
|
|
representative_title=it.title,
|
|
source_urls=[it.url] if it.url else [],
|
|
sources=[it.source] if it.source else [],
|
|
first_seen=it.published_at,
|
|
last_updated=it.published_at,
|
|
item_count=1,
|
|
sentiment_avg=it.sentiment,
|
|
item_ids=[it.id],
|
|
)
|
|
)
|
|
continue
|
|
|
|
sigs = [_minhash_signature(_shingles(f"{it.title} {it.body}")) for it in group]
|
|
labels = _dbscan(sigs, eps=eps, min_samples=min_samples)
|
|
# Singletons (label == -1) still become stories
|
|
clusters: dict[int, list[int]] = {}
|
|
for idx, lbl in enumerate(labels):
|
|
clusters.setdefault(lbl if lbl != -1 else idx, []).append(idx)
|
|
for _cid, indices in clusters.items():
|
|
members = [group[i] for i in indices]
|
|
# Pick representative = longest title (usually the most descriptive)
|
|
rep = max(members, key=lambda x: len(x.title))
|
|
sentiments = [m.sentiment for m in members if m.sentiment is not None]
|
|
avg_sent = sum(sentiments) / len(sentiments) if sentiments else 0.0
|
|
cluster_id = hashlib.sha1(
|
|
":".join(sorted(m.id for m in members)).encode()
|
|
).hexdigest()[:16]
|
|
stories.append(
|
|
StoryCluster(
|
|
cluster_id=cluster_id,
|
|
representative_title=rep.title,
|
|
source_urls=[m.url for m in members if m.url],
|
|
sources=sorted({m.source for m in members if m.source}),
|
|
first_seen=min(m.published_at for m in members),
|
|
last_updated=max(m.published_at for m in members),
|
|
item_count=len(members),
|
|
sentiment_avg=avg_sent,
|
|
item_ids=[m.id for m in members],
|
|
)
|
|
)
|
|
|
|
logger.info(
|
|
"news_clustered items=%d stories=%d windows=%d elapsed_ms=%.1f",
|
|
len(items),
|
|
len(stories),
|
|
len(windows),
|
|
(time.time() - t0) * 1000,
|
|
)
|
|
return stories
|
|
|
|
|
|
# ── DB persistence (optional, lazy import) ──────────────────────────
|
|
_PG_SCHEMA_SQL = """
|
|
CREATE TABLE IF NOT EXISTS news_clusters (
|
|
cluster_id TEXT PRIMARY KEY,
|
|
representative_title TEXT NOT NULL,
|
|
first_seen TIMESTAMPTZ NOT NULL,
|
|
last_updated TIMESTAMPTZ NOT NULL,
|
|
item_count INTEGER NOT NULL DEFAULT 0,
|
|
sentiment_avg DOUBLE PRECISION NOT NULL DEFAULT 0.0,
|
|
source_urls JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
sources JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
item_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
);
|
|
CREATE INDEX IF NOT EXISTS news_clusters_last_updated_idx
|
|
ON news_clusters (last_updated DESC);
|
|
"""
|
|
|
|
|
|
async def ensure_schema() -> bool:
|
|
"""Create news_clusters table if missing. Returns True on success."""
|
|
try:
|
|
import asyncpg
|
|
|
|
from app.core.db_pool import PG_URL
|
|
|
|
conn = await asyncpg.connect(PG_URL)
|
|
try:
|
|
await conn.execute(_PG_SCHEMA_SQL)
|
|
finally:
|
|
await conn.close()
|
|
logger.info("news_clusters_schema_ready")
|
|
return True
|
|
except Exception as exc:
|
|
logger.warning("news_clusters_schema_failed err=%s", exc)
|
|
return False
|
|
|
|
|
|
async def persist_clusters(stories: list[StoryCluster]) -> int:
|
|
"""Upsert stories to Postgres. Returns rows affected."""
|
|
if not stories:
|
|
return 0
|
|
try:
|
|
import json
|
|
|
|
import asyncpg
|
|
|
|
from app.core.db_pool import PG_URL
|
|
|
|
conn = await asyncpg.connect(PG_URL)
|
|
try:
|
|
rows = [
|
|
(
|
|
s.cluster_id,
|
|
s.representative_title,
|
|
s.first_seen,
|
|
s.last_updated,
|
|
s.item_count,
|
|
s.sentiment_avg,
|
|
json.dumps(s.source_urls),
|
|
json.dumps(s.sources),
|
|
json.dumps(s.item_ids),
|
|
)
|
|
for s in stories
|
|
]
|
|
await conn.executemany(
|
|
"""
|
|
INSERT INTO news_clusters
|
|
(cluster_id, representative_title, first_seen, last_updated,
|
|
item_count, sentiment_avg, source_urls, sources, item_ids)
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9::jsonb)
|
|
ON CONFLICT (cluster_id) DO UPDATE SET
|
|
representative_title = EXCLUDED.representative_title,
|
|
first_seen = EXCLUDED.first_seen,
|
|
last_updated = EXCLUDED.last_updated,
|
|
item_count = EXCLUDED.item_count,
|
|
sentiment_avg = EXCLUDED.sentiment_avg,
|
|
source_urls = EXCLUDED.source_urls,
|
|
sources = EXCLUDED.sources,
|
|
item_ids = EXCLUDED.item_ids
|
|
""",
|
|
rows,
|
|
)
|
|
return len(rows)
|
|
finally:
|
|
await conn.close()
|
|
except Exception as exc:
|
|
logger.warning("news_clusters_persist_failed err=%s", exc)
|
|
return 0
|
|
|
|
|
|
async def load_recent_clusters(limit: int = 50) -> list[dict]:
|
|
"""Load recent clusters from Postgres."""
|
|
try:
|
|
import json
|
|
|
|
import asyncpg
|
|
|
|
from app.core.db_pool import PG_URL
|
|
|
|
conn = await asyncpg.connect(PG_URL)
|
|
try:
|
|
rows = await conn.fetch(
|
|
"SELECT * FROM news_clusters ORDER BY last_updated DESC LIMIT $1",
|
|
limit,
|
|
)
|
|
return [
|
|
{
|
|
"cluster_id": r["cluster_id"],
|
|
"representative_title": r["representative_title"],
|
|
"first_seen": r["first_seen"].isoformat(),
|
|
"last_updated": r["last_updated"].isoformat(),
|
|
"item_count": r["item_count"],
|
|
"sentiment_avg": r["sentiment_avg"],
|
|
"source_urls": json.loads(r["source_urls"]),
|
|
"sources": json.loads(r["sources"]),
|
|
"item_ids": json.loads(r["item_ids"]),
|
|
}
|
|
for r in rows
|
|
]
|
|
finally:
|
|
await conn.close()
|
|
except Exception as exc:
|
|
logger.warning("news_clusters_load_failed err=%s", exc)
|
|
return []
|
|
|
|
|
|
__all__ = [
|
|
"NewsItem",
|
|
"StoryCluster",
|
|
"cluster_items",
|
|
"ensure_schema",
|
|
"load_recent_clusters",
|
|
"persist_clusters",
|
|
]
|