merge: chore/cleanup-remove-bloat-and-secrets into main
This commit is contained in:
commit
bde2f3a97d
1173 changed files with 437609 additions and 0 deletions
415
app/domain/news/clusterer.py
Normal file
415
app/domain/news/clusterer.py
Normal file
|
|
@ -0,0 +1,415 @@
|
|||
"""T03 — News story clustering (G04 FIX).
|
||||
|
||||
Per MINIMAX_M3_TASKS.md T03. MinHash + DBSCAN dedupes raw RSS items into
|
||||
single "stories" so AI agents don't see the same CoinDesk/The Block story
|
||||
counted 2-3x in their signal.
|
||||
|
||||
Algorithm:
|
||||
1. MinHash signature (128 permutations) on shingled title+body (first 500 chars)
|
||||
2. DBSCAN clusters within 30-minute windows, Jaccard threshold 0.6, eps=0.15
|
||||
3. Each cluster = one story with all source URLs, sentiment avg, item count
|
||||
4. Persist clusters to Postgres `news_clusters` table; raw items unchanged
|
||||
|
||||
Endpoints:
|
||||
GET /api/v1/news?clustered=true returns stories (clusters), not raw items
|
||||
GET /api/v1/news raw items (legacy)
|
||||
|
||||
This module is pure logic — no I/O at import time. Router/background job
|
||||
call `cluster_items(items) -> list[StoryCluster]`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Tokenization ────────────────────────────────────────────────────
|
||||
_TOKEN_RE = re.compile(r"[a-z0-9]{3,}", re.IGNORECASE)
|
||||
|
||||
|
||||
def _shingles(text: str, k: int = 3) -> set[str]:
|
||||
"""k-shingle set of lowercased alphanumeric tokens. For Jaccard."""
|
||||
if not text:
|
||||
return set()
|
||||
toks = _TOKEN_RE.findall(text.lower())
|
||||
if len(toks) < k:
|
||||
return set(toks)
|
||||
return {" ".join(toks[i : i + k]) for i in range(len(toks) - k + 1)}
|
||||
|
||||
|
||||
# ── MinHash ─────────────────────────────────────────────────────────
|
||||
_NUM_PERM = 128
|
||||
_MAX_HASH = (1 << 32) - 1
|
||||
|
||||
|
||||
def _minhash_signature(shingles: set[str], seed: int = 42) -> list[int]:
|
||||
"""128-permutation MinHash signature of a shingle set.
|
||||
|
||||
Uses SHA-256 seeded permutations — fast, deterministic, no numpy.
|
||||
"""
|
||||
if not shingles:
|
||||
return [_MAX_HASH] * _NUM_PERM
|
||||
sig: list[int] = []
|
||||
for i in range(_NUM_PERM):
|
||||
m = _MAX_HASH
|
||||
for s in shingles:
|
||||
h = int.from_bytes(
|
||||
hashlib.sha256(f"{i}:{seed}:{s}".encode()).digest()[:4],
|
||||
"big",
|
||||
)
|
||||
if h < m:
|
||||
m = h
|
||||
sig.append(m)
|
||||
return sig
|
||||
|
||||
|
||||
def _jaccard_minhash(a: list[int], b: list[int]) -> float:
|
||||
"""Estimate Jaccard similarity from two MinHash signatures."""
|
||||
if not a or not b or len(a) != len(b):
|
||||
return 0.0
|
||||
return sum(1 for x, y in zip(a, b, strict=False) if x == y) / len(a)
|
||||
|
||||
|
||||
# ── DBSCAN (pure-python, no sklearn dep) ────────────────────────────
|
||||
def _dbscan(
|
||||
signatures: list[list[int]],
|
||||
eps: float = 0.4,
|
||||
min_samples: int = 2,
|
||||
) -> list[int]:
|
||||
"""Density-based clustering. Returns cluster id per item (-1 = noise).
|
||||
|
||||
Similarity is Jaccard (estimated via MinHash). Neighbours are pairs
|
||||
with Jaccard distance <= eps (i.e. similarity >= 1 - eps).
|
||||
Default eps=0.4 means similarity >= 0.6 (per T03 spec).
|
||||
"""
|
||||
n = len(signatures)
|
||||
labels = [-1] * n
|
||||
cluster_id = 0
|
||||
for i in range(n):
|
||||
if labels[i] != -1:
|
||||
continue
|
||||
neighbors = [
|
||||
j
|
||||
for j in range(n)
|
||||
if i != j and (1.0 - _jaccard_minhash(signatures[i], signatures[j])) <= eps
|
||||
]
|
||||
if len(neighbors) < min_samples - 1:
|
||||
# not enough neighbours — mark as noise (may become border later)
|
||||
continue
|
||||
labels[i] = cluster_id
|
||||
seed_set = list(neighbors)
|
||||
k = 0
|
||||
while k < len(seed_set):
|
||||
q = seed_set[k]
|
||||
if labels[q] == -1:
|
||||
labels[q] = cluster_id
|
||||
q_neighbors = [
|
||||
j
|
||||
for j in range(n)
|
||||
if j != q
|
||||
and (1.0 - _jaccard_minhash(signatures[q], signatures[j])) <= eps
|
||||
]
|
||||
if len(q_neighbors) >= min_samples - 1:
|
||||
seed_set.extend(q_neighbors)
|
||||
elif labels[q] is None or labels[q] == -1:
|
||||
labels[q] = cluster_id
|
||||
k += 1
|
||||
cluster_id += 1
|
||||
return labels
|
||||
|
||||
|
||||
# ── Domain types ────────────────────────────────────────────────────
|
||||
@dataclass
|
||||
class NewsItem:
|
||||
"""Minimal news item for clustering. Adapts from DB rows or dicts."""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
body: str = ""
|
||||
source: str = ""
|
||||
url: str = ""
|
||||
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
sentiment: float = 0.0
|
||||
|
||||
@classmethod
|
||||
def from_row(cls, row: dict) -> NewsItem:
|
||||
published = row.get("published_at") or row.get("created_at")
|
||||
if isinstance(published, str):
|
||||
try:
|
||||
published = datetime.fromisoformat(published.replace("Z", "+00:00"))
|
||||
except (ValueError, AttributeError):
|
||||
published = datetime.now(UTC)
|
||||
elif not isinstance(published, datetime):
|
||||
published = datetime.now(UTC)
|
||||
return cls(
|
||||
id=str(row.get("id", row.get("news_id", ""))),
|
||||
title=row.get("title", "") or "",
|
||||
body=(row.get("body") or row.get("summary") or "")[:500],
|
||||
source=row.get("source", "") or "",
|
||||
url=row.get("url", "") or "",
|
||||
published_at=published,
|
||||
sentiment=float(row.get("sentiment", 0.0) or 0.0),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoryCluster:
|
||||
"""One deduplicated story spanning 1+ source items."""
|
||||
|
||||
cluster_id: str
|
||||
representative_title: str
|
||||
source_urls: list[str]
|
||||
sources: list[str]
|
||||
first_seen: datetime
|
||||
last_updated: datetime
|
||||
item_count: int
|
||||
sentiment_avg: float
|
||||
item_ids: list[str]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"cluster_id": self.cluster_id,
|
||||
"representative_title": self.representative_title,
|
||||
"source_urls": self.source_urls,
|
||||
"sources": self.sources,
|
||||
"first_seen": self.first_seen.isoformat(),
|
||||
"last_updated": self.last_updated.isoformat(),
|
||||
"item_count": self.item_count,
|
||||
"sentiment_avg": round(self.sentiment_avg, 3),
|
||||
"item_ids": self.item_ids,
|
||||
}
|
||||
|
||||
|
||||
# ── Main entry point ────────────────────────────────────────────────
|
||||
def cluster_items(
|
||||
items: list[NewsItem],
|
||||
window_minutes: int = 30,
|
||||
eps: float = 0.4,
|
||||
min_samples: int = 2,
|
||||
) -> list[StoryCluster]:
|
||||
"""Cluster news items into stories.
|
||||
|
||||
Items are first grouped by 30-minute time windows, then DBSCAN runs
|
||||
on MinHash signatures within each window. Single-item clusters are
|
||||
kept (they're "noise" in DBSCAN terms but valid singleton stories).
|
||||
|
||||
`eps` is the Jaccard DISTANCE threshold (1 - similarity). Per the
|
||||
task spec, two items cluster together when Jaccard similarity >= 0.6,
|
||||
so distance <= 0.4, so eps=0.4. Tighten for stricter clusters.
|
||||
"""
|
||||
t0 = time.time()
|
||||
if not items:
|
||||
return []
|
||||
|
||||
# Group by time window
|
||||
windows: dict[datetime, list[NewsItem]] = {}
|
||||
for it in sorted(items, key=lambda x: x.published_at):
|
||||
bucket = it.published_at.replace(
|
||||
minute=(it.published_at.minute // window_minutes) * window_minutes,
|
||||
second=0,
|
||||
microsecond=0,
|
||||
)
|
||||
windows.setdefault(bucket, []).append(it)
|
||||
|
||||
stories: list[StoryCluster] = []
|
||||
for _bucket, group in windows.items():
|
||||
if len(group) == 1:
|
||||
# singleton — still a story
|
||||
it = group[0]
|
||||
stories.append(
|
||||
StoryCluster(
|
||||
cluster_id=hashlib.sha1(
|
||||
f"single:{it.id}:{it.published_at.isoformat()}".encode()
|
||||
).hexdigest()[:16],
|
||||
representative_title=it.title,
|
||||
source_urls=[it.url] if it.url else [],
|
||||
sources=[it.source] if it.source else [],
|
||||
first_seen=it.published_at,
|
||||
last_updated=it.published_at,
|
||||
item_count=1,
|
||||
sentiment_avg=it.sentiment,
|
||||
item_ids=[it.id],
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
sigs = [_minhash_signature(_shingles(f"{it.title} {it.body}")) for it in group]
|
||||
labels = _dbscan(sigs, eps=eps, min_samples=min_samples)
|
||||
# Singletons (label == -1) still become stories
|
||||
clusters: dict[int, list[int]] = {}
|
||||
for idx, lbl in enumerate(labels):
|
||||
clusters.setdefault(lbl if lbl != -1 else idx, []).append(idx)
|
||||
for _cid, indices in clusters.items():
|
||||
members = [group[i] for i in indices]
|
||||
# Pick representative = longest title (usually the most descriptive)
|
||||
rep = max(members, key=lambda x: len(x.title))
|
||||
sentiments = [m.sentiment for m in members if m.sentiment is not None]
|
||||
avg_sent = sum(sentiments) / len(sentiments) if sentiments else 0.0
|
||||
cluster_id = hashlib.sha1(
|
||||
":".join(sorted(m.id for m in members)).encode()
|
||||
).hexdigest()[:16]
|
||||
stories.append(
|
||||
StoryCluster(
|
||||
cluster_id=cluster_id,
|
||||
representative_title=rep.title,
|
||||
source_urls=[m.url for m in members if m.url],
|
||||
sources=sorted({m.source for m in members if m.source}),
|
||||
first_seen=min(m.published_at for m in members),
|
||||
last_updated=max(m.published_at for m in members),
|
||||
item_count=len(members),
|
||||
sentiment_avg=avg_sent,
|
||||
item_ids=[m.id for m in members],
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"news_clustered items=%d stories=%d windows=%d elapsed_ms=%.1f",
|
||||
len(items),
|
||||
len(stories),
|
||||
len(windows),
|
||||
(time.time() - t0) * 1000,
|
||||
)
|
||||
return stories
|
||||
|
||||
|
||||
# ── DB persistence (optional, lazy import) ──────────────────────────
|
||||
_PG_SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS news_clusters (
|
||||
cluster_id TEXT PRIMARY KEY,
|
||||
representative_title TEXT NOT NULL,
|
||||
first_seen TIMESTAMPTZ NOT NULL,
|
||||
last_updated TIMESTAMPTZ NOT NULL,
|
||||
item_count INTEGER NOT NULL DEFAULT 0,
|
||||
sentiment_avg DOUBLE PRECISION NOT NULL DEFAULT 0.0,
|
||||
source_urls JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
sources JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
item_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS news_clusters_last_updated_idx
|
||||
ON news_clusters (last_updated DESC);
|
||||
"""
|
||||
|
||||
|
||||
async def ensure_schema() -> bool:
|
||||
"""Create news_clusters table if missing. Returns True on success."""
|
||||
try:
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
await conn.execute(_PG_SCHEMA_SQL)
|
||||
finally:
|
||||
await conn.close()
|
||||
logger.info("news_clusters_schema_ready")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("news_clusters_schema_failed err=%s", exc)
|
||||
return False
|
||||
|
||||
|
||||
async def persist_clusters(stories: list[StoryCluster]) -> int:
|
||||
"""Upsert stories to Postgres. Returns rows affected."""
|
||||
if not stories:
|
||||
return 0
|
||||
try:
|
||||
import json
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
rows = [
|
||||
(
|
||||
s.cluster_id,
|
||||
s.representative_title,
|
||||
s.first_seen,
|
||||
s.last_updated,
|
||||
s.item_count,
|
||||
s.sentiment_avg,
|
||||
json.dumps(s.source_urls),
|
||||
json.dumps(s.sources),
|
||||
json.dumps(s.item_ids),
|
||||
)
|
||||
for s in stories
|
||||
]
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO news_clusters
|
||||
(cluster_id, representative_title, first_seen, last_updated,
|
||||
item_count, sentiment_avg, source_urls, sources, item_ids)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9::jsonb)
|
||||
ON CONFLICT (cluster_id) DO UPDATE SET
|
||||
representative_title = EXCLUDED.representative_title,
|
||||
first_seen = EXCLUDED.first_seen,
|
||||
last_updated = EXCLUDED.last_updated,
|
||||
item_count = EXCLUDED.item_count,
|
||||
sentiment_avg = EXCLUDED.sentiment_avg,
|
||||
source_urls = EXCLUDED.source_urls,
|
||||
sources = EXCLUDED.sources,
|
||||
item_ids = EXCLUDED.item_ids
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
return len(rows)
|
||||
finally:
|
||||
await conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("news_clusters_persist_failed err=%s", exc)
|
||||
return 0
|
||||
|
||||
|
||||
async def load_recent_clusters(limit: int = 50) -> list[dict]:
|
||||
"""Load recent clusters from Postgres."""
|
||||
try:
|
||||
import json
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
rows = await conn.fetch(
|
||||
"SELECT * FROM news_clusters ORDER BY last_updated DESC LIMIT $1",
|
||||
limit,
|
||||
)
|
||||
return [
|
||||
{
|
||||
"cluster_id": r["cluster_id"],
|
||||
"representative_title": r["representative_title"],
|
||||
"first_seen": r["first_seen"].isoformat(),
|
||||
"last_updated": r["last_updated"].isoformat(),
|
||||
"item_count": r["item_count"],
|
||||
"sentiment_avg": r["sentiment_avg"],
|
||||
"source_urls": json.loads(r["source_urls"]),
|
||||
"sources": json.loads(r["sources"]),
|
||||
"item_ids": json.loads(r["item_ids"]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
finally:
|
||||
await conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("news_clusters_load_failed err=%s", exc)
|
||||
return []
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NewsItem",
|
||||
"StoryCluster",
|
||||
"cluster_items",
|
||||
"ensure_schema",
|
||||
"load_recent_clusters",
|
||||
"persist_clusters",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue