cryptorugmunch
e60a62a07a
Follow-up to the BLE001 refactor. The auto-conversion of except Exception -> except (httpx.HTTPError, httpx.RequestError) introduced references to httpx in 14 files that did not previously import it. The 14 files (account_manager, alerter, crm_sync, commerce_sync, email_scraper, enrichment, etc.) all use the shared client.py internally, so the import was missing but not strictly broken. Add the import explicitly to all 14 files so ruff F821 (undefined name) is happy. Existing behavior is preserved.
235 lines
7.7 KiB
Python
235 lines
7.7 KiB
Python
import httpx
|
|
"""Pry — Adaptive Freshness Scheduling.
|
|
Conditional scraping, content fingerprinting, staleness dashboard, adaptive frequency."""
|
|
from paths import PRY_DATA_DIR
|
|
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
from contextlib import suppress
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FRESHNESS_DIR = PRY_DATA_DIR / "freshness"
|
|
FRESHNESS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
# ── Content Fingerprinting ──
|
|
|
|
|
|
def compute_content_hash(content: str) -> str:
|
|
"""Compute a stable content hash for change detection."""
|
|
normalized = " ".join(content.split()) # Normalize whitespace
|
|
return hashlib.sha256(normalized.encode()).hexdigest()[:32]
|
|
|
|
|
|
async def check_content_changed(url: str, content: str) -> dict[str, Any]:
|
|
"""Check if content has changed since last scrape using content hash.
|
|
|
|
Returns:
|
|
changed: bool — whether content is different from last known
|
|
previous_hash: str — hash of previous content
|
|
current_hash: str — hash of current content
|
|
last_changed: str — ISO timestamp of last detected change
|
|
"""
|
|
url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
|
|
fingerprint_path = FRESHNESS_DIR / f"fingerprint_{url_hash}.json"
|
|
current_hash = compute_content_hash(content)
|
|
|
|
result: dict[str, Any] = {
|
|
"url": url,
|
|
"current_hash": current_hash,
|
|
"previous_hash": None,
|
|
"changed": True,
|
|
"last_changed": datetime.now(UTC).isoformat(),
|
|
"last_checked": datetime.now(UTC).isoformat(),
|
|
"is_new": True,
|
|
}
|
|
|
|
if fingerprint_path.exists():
|
|
try:
|
|
previous = json.loads(fingerprint_path.read_text())
|
|
result["previous_hash"] = previous.get("hash")
|
|
result["last_changed"] = previous.get("last_changed", "")
|
|
result["is_new"] = False
|
|
result["changed"] = current_hash != previous.get("hash")
|
|
except (json.JSONDecodeError, OSError):
|
|
pass
|
|
|
|
# Save current fingerprint
|
|
with suppress(OSError):
|
|
fingerprint_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"url": url,
|
|
"hash": current_hash,
|
|
"last_checked": result["last_checked"],
|
|
"last_changed": result["last_changed"],
|
|
}
|
|
)
|
|
)
|
|
|
|
return result
|
|
|
|
|
|
async def quick_health_check(url: str) -> dict[str, Any]:
|
|
"""Quick HEAD request to check if a URL is responsive without full scrape."""
|
|
from client import get_client
|
|
|
|
client = await get_client()
|
|
try:
|
|
resp = await client.head(url, timeout=10, follow_redirects=True)
|
|
return {
|
|
"url": url,
|
|
"status_code": resp.status_code,
|
|
"accessible": resp.is_success,
|
|
"content_type": resp.headers.get("content-type", ""),
|
|
"content_length": resp.headers.get("content-length", "0"),
|
|
"last_modified": resp.headers.get("last-modified", ""),
|
|
"etag": resp.headers.get("etag", ""),
|
|
}
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
return {"url": url, "accessible": False, "error": str(e)[:100]}
|
|
|
|
|
|
# ── Adaptive Frequency Calculation ──
|
|
|
|
|
|
def calculate_adaptive_frequency(
|
|
url: str,
|
|
base_interval_minutes: int = 60,
|
|
min_interval: int = 15,
|
|
max_interval: int = 1440, # 24h
|
|
volatility_window: int = 10, # Number of checks to look back
|
|
) -> dict[str, Any]:
|
|
"""Calculate optimal scrape frequency based on content change history.
|
|
|
|
Uses a simple Bayesian approach: if content changes frequently,
|
|
increase frequency. If stable, decrease frequency.
|
|
"""
|
|
url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
|
|
history_path = FRESHNESS_DIR / f"history_{url_hash}.json"
|
|
|
|
changes = 0
|
|
total_checks = 0
|
|
change_history: list[bool] = []
|
|
|
|
if history_path.exists():
|
|
try:
|
|
history = json.loads(history_path.read_text())
|
|
change_history = history.get("changes", [])[-volatility_window:]
|
|
total_checks = len(change_history)
|
|
changes = sum(1 for c in change_history if c)
|
|
except (json.JSONDecodeError, OSError):
|
|
pass
|
|
|
|
# Store current check
|
|
# (this is called after a scrape, so we record the result)
|
|
|
|
# Compute change rate
|
|
change_rate = changes / max(total_checks, 1)
|
|
|
|
# Adjust interval
|
|
if change_rate > 0.3:
|
|
# Volatile — increase frequency
|
|
interval = max(min_interval, int(base_interval_minutes * (1 - change_rate)))
|
|
elif change_rate < 0.05 and total_checks >= 5:
|
|
# Very stable — decrease frequency
|
|
interval = min(max_interval, int(base_interval_minutes * 2))
|
|
else:
|
|
interval = base_interval_minutes
|
|
|
|
return {
|
|
"url": url,
|
|
"suggested_interval_minutes": interval,
|
|
"change_rate": round(change_rate, 3),
|
|
"total_checks_history": total_checks,
|
|
"changes_detected": changes,
|
|
"volatility": "high" if change_rate > 0.3 else "medium" if change_rate > 0.1 else "low",
|
|
"base_interval": base_interval_minutes,
|
|
}
|
|
|
|
|
|
def record_check_result(url: str, changed: bool) -> None:
|
|
"""Record a check result for adaptive frequency calculation."""
|
|
url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
|
|
history_path = FRESHNESS_DIR / f"history_{url_hash}.json"
|
|
|
|
history: dict[str, Any] = {"url": url, "changes": []}
|
|
if history_path.exists():
|
|
with suppress(json.JSONDecodeError, OSError):
|
|
history = json.loads(history_path.read_text())
|
|
|
|
history["changes"].append(changed)
|
|
history["last_updated"] = datetime.now(UTC).isoformat()
|
|
|
|
# Keep only last 100 entries
|
|
if len(history["changes"]) > 100:
|
|
history["changes"] = history["changes"][-100:]
|
|
|
|
with suppress(OSError):
|
|
history_path.write_text(json.dumps(history))
|
|
|
|
|
|
# ── Staleness Dashboard ──
|
|
|
|
|
|
def get_staleness_dashboard() -> dict[str, Any]:
|
|
"""Get the staleness dashboard showing all tracked URLs and their freshness."""
|
|
urls: list[dict[str, Any]] = []
|
|
stale_count = 0
|
|
max_age_hours = 24
|
|
|
|
for path in FRESHNESS_DIR.glob("fingerprint_*.json"):
|
|
try:
|
|
data = json.loads(path.read_text())
|
|
last_checked = data.get("last_checked", "")
|
|
last_changed = data.get("last_changed", "")
|
|
url = data.get("url", "")
|
|
|
|
age_hours = 0.0
|
|
if last_checked:
|
|
try:
|
|
checked_dt = datetime.fromisoformat(last_checked)
|
|
age_hours = (datetime.now(UTC) - checked_dt).total_seconds() / 3600
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
is_stale = age_hours > max_age_hours
|
|
|
|
urls.append(
|
|
{
|
|
"url": url[:100],
|
|
"last_checked": last_checked,
|
|
"last_changed": last_changed,
|
|
"age_hours": round(age_hours, 1),
|
|
"stale": is_stale,
|
|
"hash": data.get("hash", "")[:12],
|
|
}
|
|
)
|
|
if is_stale:
|
|
stale_count += 1
|
|
|
|
except (json.JSONDecodeError, OSError):
|
|
continue
|
|
|
|
# Sort by last_checked (oldest first)
|
|
urls.sort(key=lambda x: x.get("age_hours", 0), reverse=True)
|
|
|
|
return {
|
|
"total_tracked": len(urls),
|
|
"stale_count": stale_count,
|
|
"fresh_count": len(urls) - stale_count,
|
|
"max_age_hours": max_age_hours,
|
|
"urls": urls[:100], # Limit to 100
|
|
}
|