pryscraper/freshness.py

"""Pry — Adaptive Freshness Scheduling.
Conditional scraping, content fingerprinting, staleness dashboard, adaptive frequency."""

import hashlib
import json
import logging
import os
from contextlib import suppress
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

logger = logging.getLogger(__name__)

FRESHNESS_DIR = Path(os.path.expanduser("~/.pry/freshness"))
FRESHNESS_DIR.mkdir(parents=True, exist_ok=True)


# ── Content Fingerprinting ──


def compute_content_hash(content: str) -> str:
    """Compute a stable content hash for change detection."""
    normalized = " ".join(content.split())  # Normalize whitespace
    return hashlib.sha256(normalized.encode()).hexdigest()[:32]


async def check_content_changed(url: str, content: str) -> dict[str, Any]:
    """Check if content has changed since last scrape using content hash.

    Returns:
        changed: bool — whether content is different from last known
        previous_hash: str — hash of previous content
        current_hash: str — hash of current content
        last_changed: str — ISO timestamp of last detected change
    """
    url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
    fingerprint_path = FRESHNESS_DIR / f"fingerprint_{url_hash}.json"
    current_hash = compute_content_hash(content)

    result: dict[str, Any] = {
        "url": url,
        "current_hash": current_hash,
        "previous_hash": None,
        "changed": True,
        "last_changed": datetime.now(UTC).isoformat(),
        "last_checked": datetime.now(UTC).isoformat(),
        "is_new": True,
    }

    if fingerprint_path.exists():
        try:
            previous = json.loads(fingerprint_path.read_text())
            result["previous_hash"] = previous.get("hash")
            result["last_changed"] = previous.get("last_changed", "")
            result["is_new"] = False
            result["changed"] = current_hash != previous.get("hash")
        except (json.JSONDecodeError, OSError):
            pass

    # Save current fingerprint
    with suppress(OSError):
        fingerprint_path.write_text(
            json.dumps(
                {
                    "url": url,
                    "hash": current_hash,
                    "last_checked": result["last_checked"],
                    "last_changed": result["last_changed"],
                }
            )
        )

    return result


async def quick_health_check(url: str) -> dict[str, Any]:
    """Quick HEAD request to check if a URL is responsive without full scrape."""
    from client import get_client

    client = await get_client()
    try:
        resp = await client.head(url, timeout=10, follow_redirects=True)
        return {
            "url": url,
            "status_code": resp.status_code,
            "accessible": resp.is_success,
            "content_type": resp.headers.get("content-type", ""),
            "content_length": resp.headers.get("content-length", "0"),
            "last_modified": resp.headers.get("last-modified", ""),
            "etag": resp.headers.get("etag", ""),
        }
    except Exception as e:
        return {"url": url, "accessible": False, "error": str(e)[:100]}


# ── Adaptive Frequency Calculation ──


def calculate_adaptive_frequency(
    url: str,
    base_interval_minutes: int = 60,
    min_interval: int = 15,
    max_interval: int = 1440,  # 24h
    volatility_window: int = 10,  # Number of checks to look back
) -> dict[str, Any]:
    """Calculate optimal scrape frequency based on content change history.

    Uses a simple Bayesian approach: if content changes frequently,
    increase frequency. If stable, decrease frequency.
    """
    url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
    history_path = FRESHNESS_DIR / f"history_{url_hash}.json"

    changes = 0
    total_checks = 0
    change_history: list[bool] = []

    if history_path.exists():
        try:
            history = json.loads(history_path.read_text())
            change_history = history.get("changes", [])[-volatility_window:]
            total_checks = len(change_history)
            changes = sum(1 for c in change_history if c)
        except (json.JSONDecodeError, OSError):
            pass

    # Store current check
    # (this is called after a scrape, so we record the result)

    # Compute change rate
    change_rate = changes / max(total_checks, 1)

    # Adjust interval
    if change_rate > 0.3:
        # Volatile — increase frequency
        interval = max(min_interval, int(base_interval_minutes * (1 - change_rate)))
    elif change_rate < 0.05 and total_checks >= 5:
        # Very stable — decrease frequency
        interval = min(max_interval, int(base_interval_minutes * 2))
    else:
        interval = base_interval_minutes

    return {
        "url": url,
        "suggested_interval_minutes": interval,
        "change_rate": round(change_rate, 3),
        "total_checks_history": total_checks,
        "changes_detected": changes,
        "volatility": "high" if change_rate > 0.3 else "medium" if change_rate > 0.1 else "low",
        "base_interval": base_interval_minutes,
    }


def record_check_result(url: str, changed: bool) -> None:
    """Record a check result for adaptive frequency calculation."""
    url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
    history_path = FRESHNESS_DIR / f"history_{url_hash}.json"

    history: dict[str, Any] = {"url": url, "changes": []}
    if history_path.exists():
        with suppress(json.JSONDecodeError, OSError):
            history = json.loads(history_path.read_text())

    history["changes"].append(changed)
    history["last_updated"] = datetime.now(UTC).isoformat()

    # Keep only last 100 entries
    if len(history["changes"]) > 100:
        history["changes"] = history["changes"][-100:]

    with suppress(OSError):
        history_path.write_text(json.dumps(history))


# ── Staleness Dashboard ──


def get_staleness_dashboard() -> dict[str, Any]:
    """Get the staleness dashboard showing all tracked URLs and their freshness."""
    urls: list[dict[str, Any]] = []
    stale_count = 0
    max_age_hours = 24

    for path in FRESHNESS_DIR.glob("fingerprint_*.json"):
        try:
            data = json.loads(path.read_text())
            last_checked = data.get("last_checked", "")
            last_changed = data.get("last_changed", "")
            url = data.get("url", "")

            age_hours = 0.0
            if last_checked:
                try:
                    checked_dt = datetime.fromisoformat(last_checked)
                    age_hours = (datetime.now(UTC) - checked_dt).total_seconds() / 3600
                except (ValueError, TypeError):
                    pass

            is_stale = age_hours > max_age_hours

            urls.append(
                {
                    "url": url[:100],
                    "last_checked": last_checked,
                    "last_changed": last_changed,
                    "age_hours": round(age_hours, 1),
                    "stale": is_stale,
                    "hash": data.get("hash", "")[:12],
                }
            )
            if is_stale:
                stale_count += 1

        except (json.JSONDecodeError, OSError):
            continue

    # Sort by last_checked (oldest first)
    urls.sort(key=lambda x: x.get("age_hours", 0), reverse=True)

    return {
        "total_tracked": len(urls),
        "stale_count": stale_count,
        "fresh_count": len(urls) - stale_count,
        "max_age_hours": max_age_hours,
        "urls": urls[:100],  # Limit to 100
    }