"""Pry — Adaptive Freshness Scheduling. Conditional scraping, content fingerprinting, staleness dashboard, adaptive frequency.""" import hashlib import json import logging import os from contextlib import suppress from datetime import UTC, datetime from pathlib import Path from typing import Any logger = logging.getLogger(__name__) FRESHNESS_DIR = Path(os.path.expanduser("~/.pry/freshness")) FRESHNESS_DIR.mkdir(parents=True, exist_ok=True) # ── Content Fingerprinting ── def compute_content_hash(content: str) -> str: """Compute a stable content hash for change detection.""" normalized = " ".join(content.split()) # Normalize whitespace return hashlib.sha256(normalized.encode()).hexdigest()[:32] async def check_content_changed(url: str, content: str) -> dict[str, Any]: """Check if content has changed since last scrape using content hash. Returns: changed: bool — whether content is different from last known previous_hash: str — hash of previous content current_hash: str — hash of current content last_changed: str — ISO timestamp of last detected change """ url_hash = hashlib.sha256(url.encode()).hexdigest()[:16] fingerprint_path = FRESHNESS_DIR / f"fingerprint_{url_hash}.json" current_hash = compute_content_hash(content) result: dict[str, Any] = { "url": url, "current_hash": current_hash, "previous_hash": None, "changed": True, "last_changed": datetime.now(UTC).isoformat(), "last_checked": datetime.now(UTC).isoformat(), "is_new": True, } if fingerprint_path.exists(): try: previous = json.loads(fingerprint_path.read_text()) result["previous_hash"] = previous.get("hash") result["last_changed"] = previous.get("last_changed", "") result["is_new"] = False result["changed"] = current_hash != previous.get("hash") except (json.JSONDecodeError, OSError): pass # Save current fingerprint with suppress(OSError): fingerprint_path.write_text( json.dumps( { "url": url, "hash": current_hash, "last_checked": result["last_checked"], "last_changed": result["last_changed"], } ) ) return result async def quick_health_check(url: str) -> dict[str, Any]: """Quick HEAD request to check if a URL is responsive without full scrape.""" from client import get_client client = await get_client() try: resp = await client.head(url, timeout=10, follow_redirects=True) return { "url": url, "status_code": resp.status_code, "accessible": resp.is_success, "content_type": resp.headers.get("content-type", ""), "content_length": resp.headers.get("content-length", "0"), "last_modified": resp.headers.get("last-modified", ""), "etag": resp.headers.get("etag", ""), } except Exception as e: return {"url": url, "accessible": False, "error": str(e)[:100]} # ── Adaptive Frequency Calculation ── def calculate_adaptive_frequency( url: str, base_interval_minutes: int = 60, min_interval: int = 15, max_interval: int = 1440, # 24h volatility_window: int = 10, # Number of checks to look back ) -> dict[str, Any]: """Calculate optimal scrape frequency based on content change history. Uses a simple Bayesian approach: if content changes frequently, increase frequency. If stable, decrease frequency. """ url_hash = hashlib.sha256(url.encode()).hexdigest()[:16] history_path = FRESHNESS_DIR / f"history_{url_hash}.json" changes = 0 total_checks = 0 change_history: list[bool] = [] if history_path.exists(): try: history = json.loads(history_path.read_text()) change_history = history.get("changes", [])[-volatility_window:] total_checks = len(change_history) changes = sum(1 for c in change_history if c) except (json.JSONDecodeError, OSError): pass # Store current check # (this is called after a scrape, so we record the result) # Compute change rate change_rate = changes / max(total_checks, 1) # Adjust interval if change_rate > 0.3: # Volatile — increase frequency interval = max(min_interval, int(base_interval_minutes * (1 - change_rate))) elif change_rate < 0.05 and total_checks >= 5: # Very stable — decrease frequency interval = min(max_interval, int(base_interval_minutes * 2)) else: interval = base_interval_minutes return { "url": url, "suggested_interval_minutes": interval, "change_rate": round(change_rate, 3), "total_checks_history": total_checks, "changes_detected": changes, "volatility": "high" if change_rate > 0.3 else "medium" if change_rate > 0.1 else "low", "base_interval": base_interval_minutes, } def record_check_result(url: str, changed: bool) -> None: """Record a check result for adaptive frequency calculation.""" url_hash = hashlib.sha256(url.encode()).hexdigest()[:16] history_path = FRESHNESS_DIR / f"history_{url_hash}.json" history: dict[str, Any] = {"url": url, "changes": []} if history_path.exists(): with suppress(json.JSONDecodeError, OSError): history = json.loads(history_path.read_text()) history["changes"].append(changed) history["last_updated"] = datetime.now(UTC).isoformat() # Keep only last 100 entries if len(history["changes"]) > 100: history["changes"] = history["changes"][-100:] with suppress(OSError): history_path.write_text(json.dumps(history)) # ── Staleness Dashboard ── def get_staleness_dashboard() -> dict[str, Any]: """Get the staleness dashboard showing all tracked URLs and their freshness.""" urls: list[dict[str, Any]] = [] stale_count = 0 max_age_hours = 24 for path in FRESHNESS_DIR.glob("fingerprint_*.json"): try: data = json.loads(path.read_text()) last_checked = data.get("last_checked", "") last_changed = data.get("last_changed", "") url = data.get("url", "") age_hours = 0.0 if last_checked: try: checked_dt = datetime.fromisoformat(last_checked) age_hours = (datetime.now(UTC) - checked_dt).total_seconds() / 3600 except (ValueError, TypeError): pass is_stale = age_hours > max_age_hours urls.append( { "url": url[:100], "last_checked": last_checked, "last_changed": last_changed, "age_hours": round(age_hours, 1), "stale": is_stale, "hash": data.get("hash", "")[:12], } ) if is_stale: stale_count += 1 except (json.JSONDecodeError, OSError): continue # Sort by last_checked (oldest first) urls.sort(key=lambda x: x.get("age_hours", 0), reverse=True) return { "total_tracked": len(urls), "stale_count": stale_count, "fresh_count": len(urls) - stale_count, "max_age_hours": max_age_hours, "urls": urls[:100], # Limit to 100 }