feat(sql): migrate intelligence, monitor, and referrals to SQLAlchemy (#4)

db.py

@@ -257,7 +257,16 @@ if _HAS_SA:
         __tablename__ = "structure_snapshots"
         id = Column(Integer, primary_key=True)
         url = Column(Text, index=True, nullable=False)
+        url_hash = Column(String(64), default="", index=True)
+        template_id = Column(String(128), default="", index=True)
         structure_hash = Column(String(64), default="")
+        selectors = Column(JSON, default=list)
+        all_matched = Column(Boolean, default=False)
+        matched_count = Column(Integer, default=0)
+        failed_count = Column(Integer, default=0)
+        changes = Column(JSON, default=list)
+        change_count = Column(Integer, default=0)
+        has_changes = Column(Boolean, default=False)
         dom_summary = Column(JSON, default=dict)
         checked_at = Column(DateTime, default=_now, index=True)
 
@@ -283,9 +292,16 @@ if _HAS_SA:
         monitor_name = Column(String(256), default="")  # was 'name' in JSON
         schedule_cron = Column(String(64), default="")
         check_type = Column(String(32), default="content")
+        goal = Column(Text, default="")
+        use_llm_judge = Column(Boolean, default=False)
         active = Column(Boolean, default=True, index=True)
         webhook_url = Column(Text, default="")
         last_check_at = Column(DateTime, nullable=True)
+        last_run_at = Column(DateTime, nullable=True)
+        current_version = Column(Integer, default=0)
+        last_content = Column(Text, default="")
+        total_checks = Column(Integer, default=0)
+        total_changes = Column(Integer, default=0)
         created_at = Column(DateTime, default=_now)
 
     class MonitorRun(Base):
@@ -419,6 +435,24 @@ if _HAS_SA:
         converted = Column(Boolean, default=False)
         conversion_value_usd = Column(Float, default=0.0)
 
+    class ReferralConversion(Base):
+        __tablename__ = "referral_conversions"
+        id = Column(Integer, primary_key=True)
+        click_id = Column(
+            String(64), ForeignKey("referral_clicks.click_id"), index=True, nullable=False
+        )
+        provider = Column(String(64), index=True, nullable=False)
+        revenue_usd = Column(Float, default=0.0)
+        notes = Column(Text, default="")
+        ts = Column(DateTime, default=_now, index=True)
+
+    class ReferralInAppUsage(Base):
+        __tablename__ = "referral_in_app_usage"
+        id = Column(Integer, primary_key=True)
+        provider = Column(String(64), index=True, nullable=False)
+        revenue_usd = Column(Float, default=0.0)
+        ts = Column(DateTime, default=_now, index=True)
+
     # ── Actors (Apify-style marketplace) ──
     class Actor(Base):
         __tablename__ = "actors"
@@ -678,6 +712,71 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
             n += 1
         counts["agency"] = n
 
+        # Structure snapshots (JSON files)
+        n = 0
+        for rec in _iter_json_files(data_dir / "structure"):
+            s.add(
+                StructureSnapshot(
+                    url=rec.get("url", ""),
+                    url_hash=rec.get("url_hash", ""),
+                    template_id=rec.get("template_id", ""),
+                    structure_hash=rec.get("structure_hash", ""),
+                    selectors=rec.get("selectors", []),
+                    all_matched=rec.get("all_matched", False),
+                    matched_count=rec.get("matched_count", 0),
+                    failed_count=rec.get("failed_count", 0),
+                    changes=rec.get("changes", []),
+                    change_count=rec.get("change_count", 0),
+                    has_changes=rec.get("has_changes", False),
+                    dom_summary=rec.get("dom_summary", {}),
+                )
+            )
+            n += 1
+        counts["structure"] = n
+
+        # Referral clicks (JSONL)
+        n = 0
+        for rec in _iter_jsonl(data_dir / "referrals" / "clicks.jsonl"):
+            s.add(
+                ReferralClick(
+                    click_id=rec.get("id", "")[:64],
+                    provider=rec.get("provider", "")[:64],
+                    url=rec.get("url", ""),
+                    source=rec.get("source", ""),
+                    user_id=rec.get("user_id", ""),
+                    converted=rec.get("converted", False),
+                    conversion_value_usd=rec.get("conversion_value_usd", 0.0),
+                )
+            )
+            n += 1
+        counts["referral_clicks"] = n
+
+        # Referral conversions (JSONL)
+        n = 0
+        for rec in _iter_jsonl(data_dir / "referrals" / "conversions.jsonl"):
+            s.add(
+                ReferralConversion(
+                    click_id=rec.get("click_id", "")[:64],
+                    provider=rec.get("provider", "")[:64],
+                    revenue_usd=rec.get("revenue_usd", 0.0),
+                    notes=rec.get("notes", ""),
+                )
+            )
+            n += 1
+        counts["referral_conversions"] = n
+
+        # Referral in-app usage (JSONL)
+        n = 0
+        for rec in _iter_jsonl(data_dir / "referrals" / "in_app_usage.jsonl"):
+            s.add(
+                ReferralInAppUsage(
+                    provider=rec.get("provider", "")[:64],
+                    revenue_usd=rec.get("revenue_usd", 0.0),
+                )
+            )
+            n += 1
+        counts["referral_in_app_usage"] = n
+
     return counts
 
 

intelligence.py

@@ -6,28 +6,19 @@ Historical snapshots, anomaly detection, natural-language alerts, weekly reports
 #
 # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
 # Licensed under MIT. See LICENSE.
-import json
 import logging
 import statistics
 import time
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
-from pathlib import Path
 from typing import Any
 
-from paths import PRY_DATA_DIR
+from sqlalchemy import desc
+from sqlalchemy.exc import SQLAlchemyError
+
+from db import IntelSnapshot, session_scope
 
 logger = logging.getLogger(__name__)
 
-INTEL_DIR = PRY_DATA_DIR / "intel"
-INTEL_DIR.mkdir(parents=True, exist_ok=True)
-
-
-# ── Historical Snapshots ──
-
-
-def _snapshot_path(competitor_id: str) -> Path:
-    return INTEL_DIR / f"{competitor_id}_snapshots.jsonl"
-
 
 def record_snapshot(
     competitor_id: str,
@@ -35,25 +26,32 @@ def record_snapshot(
     url: str,
     fields: dict[str, Any],
 ) -> dict[str, Any]:
-    """Record a data snapshot for a competitor.
+    """Record a data snapshot for a competitor."""
-
-    Each snapshot is appended to a JSONL file for the competitor.
-    """
-    snapshot = {
-        "ts": datetime.now(UTC).isoformat(),
-        "unix_ts": time.time(),
-        "competitor_id": competitor_id,
-        "competitor_name": competitor_name,
-        "url": url,
-        "fields": fields,
-    }
-    path = _snapshot_path(competitor_id)
     try:
-        with open(path, "a") as f:
+        with session_scope() as session:
-            f.write(json.dumps(snapshot) + "\n")
+            snap = IntelSnapshot(
+                competitor_id=competitor_id,
+                competitor_name=competitor_name,
+                url=url,
+                fields=fields,
+            )
+            session.add(snap)
+            session.flush()
+            ts = snap.ts
+            result = {
+                "ts": ts.isoformat(),
+                "unix_ts": ts.timestamp(),
+                "competitor_id": competitor_id,
+                "competitor_name": competitor_name,
+                "url": url,
+                "fields": fields,
+            }
         logger.info("snapshot_recorded", extra={"competitor": competitor_name})
-        return snapshot
+        return result
-    except OSError as e:
+    except SQLAlchemyError as e:
+        logger.error(
+            "snapshot_record_failed", extra={"competitor": competitor_name, "error": str(e)}
+        )
         return {"error": str(e)}
 
 
@@ -63,31 +61,31 @@ def get_snapshots(
     since_hours: int | None = None,
 ) -> list[dict[str, Any]]:
     """Get snapshots for a competitor, most recent first."""
-    path = _snapshot_path(competitor_id)
-    if not path.exists():
-        return []
-
-    snapshots = []
     try:
-        for line in path.read_text().splitlines():
+        with session_scope() as session:
-            if not line.strip():
+            query = session.query(IntelSnapshot).filter(
-                continue
+                IntelSnapshot.competitor_id == competitor_id,
-            snapshots.append(json.loads(line))
+            )
-    except (json.JSONDecodeError, OSError):
+            if since_hours is not None:
+                cutoff = datetime.now(UTC) - timedelta(hours=since_hours)
+                query = query.filter(IntelSnapshot.ts >= cutoff)
+            query = query.order_by(desc(IntelSnapshot.ts)).limit(limit)
+            rows = query.all()
+            return [
+                {
+                    "ts": row.ts.isoformat() if row.ts else "",
+                    "unix_ts": row.ts.timestamp() if row.ts else 0,
+                    "competitor_id": row.competitor_id,
+                    "competitor_name": row.competitor_name,
+                    "url": row.url,
+                    "fields": row.fields or {},
+                }
+                for row in rows
+            ]
+    except SQLAlchemyError as e:
+        logger.error("snapshot_query_failed", extra={"competitor": competitor_id, "error": str(e)})
         return []
 
-    # Filter by time
-    if since_hours:
-        cutoff = time.time() - (since_hours * 3600)
-        snapshots = [s for s in snapshots if s.get("unix_ts", 0) >= cutoff]
-
-    # Sort by time (newest first) and limit
-    snapshots.sort(key=lambda x: x.get("unix_ts", 0), reverse=True)
-    return snapshots[:limit]
-
-
-# ── Anomaly Detection ──
-
 
 def compute_field_statistics(
     snapshots: list[dict[str, Any]],
@@ -154,9 +152,6 @@ def detect_anomalies_numeric(
     }
 
 
-# ── Natural Language Alerts ──
-
-
 def generate_alert(
     competitor_name: str,
     field: str,
@@ -195,9 +190,6 @@ def generate_alert(
     return alert
 
 
-# ── Weekly Reports ──
-
-
 def generate_weekly_report(
     competitors: list[dict[str, Any]],
     days_back: int = 7,
@@ -215,7 +207,6 @@ def generate_weekly_report(
         if not weekly:
             continue
 
-        # Get fields that changed this week
         if len(weekly) >= 2:
             latest = weekly[0].get("fields", {})
             oldest = weekly[-1].get("fields", {})
@@ -245,7 +236,6 @@ def generate_weekly_report(
                     }
                 )
 
-    # Summary
     total_changes = sum(s["changes_count"] for s in report_sections)
     most_active = (
         max(report_sections, key=lambda x: x["changes_count"]) if report_sections else None

monitor.py

@@ -9,33 +9,17 @@ Cron-based monitors that diff content and judge meaningful changes."""
 import difflib
 import json
 import logging
-import os
 import re
-import shutil
 import uuid
 from datetime import UTC, datetime
-from pathlib import Path
 from typing import Any
 
 import httpx
 
-from paths import PRY_DATA_DIR
+from db import Monitor, MonitorRun, session_scope
 
 logger = logging.getLogger(__name__)
 
-MONITORS_DIR = PRY_DATA_DIR / "monitors"
-MONITORS_DIR.mkdir(parents=True, exist_ok=True)
-
-
-def _monitor_path(monitor_id: str) -> Path:
-    return MONITORS_DIR / f"{monitor_id}.json"
-
-
-def _snapshot_path(monitor_id: str, version: int) -> Path:
-    snap_dir = MONITORS_DIR / monitor_id
-    snap_dir.mkdir(parents=True, exist_ok=True)
-    return snap_dir / f"snapshot_{version}.json"
-
 
 def _compute_diff(previous: str, current: str) -> str:
     differ = difflib.Differ()
@@ -45,6 +29,25 @@ def _compute_diff(previous: str, current: str) -> str:
     return f"{added} lines added, {removed} lines removed"
 
 
+def _monitor_to_dict(m: Monitor) -> dict[str, Any]:
+    return {
+        "id": m.monitor_id,
+        "name": m.monitor_name,
+        "target_url": m.url,
+        "schedule_cron": m.schedule_cron,
+        "goal": m.goal or "",
+        "webhook_url": m.webhook_url or "",
+        "use_llm_judge": m.use_llm_judge or False,
+        "status": "active" if m.active else "inactive",
+        "created_at": m.created_at.isoformat() if m.created_at else "",
+        "last_run_at": m.last_run_at.isoformat() if m.last_run_at else None,
+        "current_version": m.current_version or 0,
+        "last_content": m.last_content or "",
+        "total_checks": m.total_checks or 0,
+        "total_changes": m.total_changes or 0,
+    }
+
+
 class ChangeJudger:
     """Judge whether a content change is meaningful using LLM or heuristics."""
 
@@ -89,7 +92,6 @@ class ChangeJudger:
 
         goal_in_current = any(gk in curr_text_lower for gk in goal_keywords)
 
-        # Check if goal keywords appear in newly-added tokens
         def _keyword_match(tokens: set[str]) -> list[str]:
             matched: list[str] = []
             for gk in goal_keywords:
@@ -114,7 +116,6 @@ class ChangeJudger:
                 ],
             }
 
-        # Content changed and goal keywords still present -> meaningful update
         if goal_in_current and len(previous) > 0:
             change_ratio = abs(len(current) - len(previous)) / len(previous)
             if change_ratio > 0.01:
@@ -130,7 +131,6 @@ class ChangeJudger:
                     ],
                 }
 
-        # Large generic change (no goal or goal keywords absent)
         if len(previous) > 0:
             change_ratio = abs(len(current) - len(previous)) / len(previous)
             if change_ratio > 0.1:
@@ -187,7 +187,7 @@ Is this change meaningful given the goal? Respond with JSON:
             response_text = resp.json().get("response", "")
             json_match = re.search(r"\{.*\}", response_text, re.DOTALL)
             if json_match:
-                return json.loads(json_match.group(0))  # type: ignore[no-any-return]
+                return json.loads(json_match.group(0))
         except (json.JSONDecodeError, ValueError):
             logger.warning("llm_judge_failed, falling back to heuristic")
 
@@ -203,78 +203,109 @@ async def create_monitor(
     use_llm_judge: bool = False,
 ) -> dict[str, Any]:
     monitor_id = uuid.uuid4().hex[:12]
-    monitor = {
+    now = datetime.now(UTC)
-        "id": monitor_id,
+    monitor = Monitor(
-        "name": name,
+        monitor_id=monitor_id,
-        "target_url": target_url,
+        url=target_url,
-        "schedule_cron": schedule_cron,
+        monitor_name=name,
-        "goal": goal,
+        schedule_cron=schedule_cron,
-        "webhook_url": webhook_url,
+        goal=goal,
-        "use_llm_judge": use_llm_judge,
+        use_llm_judge=use_llm_judge,
-        "status": "active",
+        active=True,
-        "created_at": datetime.now(UTC).isoformat(),
+        webhook_url=webhook_url,
-        "last_run_at": None,
+        created_at=now,
-        "current_version": 0,
+        current_version=0,
-        "last_content": "",
+        last_content="",
-        "total_checks": 0,
+        total_checks=0,
-        "total_changes": 0,
+        total_changes=0,
-    }
+    )
-    path = _monitor_path(monitor_id)
+    try:
-    path.write_text(json.dumps(monitor, indent=2))
+        with session_scope() as s:
-    logger.info("monitor_created", extra={"monitor_id": monitor_id, "monitor_name": name})
+            s.add(monitor)
-    return monitor
+        logger.info("monitor_created", extra={"monitor_id": monitor_id, "monitor_name": name})
+        return _monitor_to_dict(monitor)
+    except Exception:
+        logger.exception("monitor_creation_failed", extra={"monitor_name": name})
+        raise
 
 
 async def run_monitor(monitor_id: str) -> dict[str, Any]:
-    path = _monitor_path(monitor_id)
-    if not path.exists():
-        return {"error": f"Monitor not found: {monitor_id}"}
-
-    monitor = json.loads(path.read_text())
-    previous_content = monitor.get("last_content", "")
-
     try:
-        from scraper import PryScraper
+        with session_scope() as s:
+            monitor = s.query(Monitor).filter(Monitor.monitor_id == monitor_id).first()
+            if not monitor:
+                return {"error": f"Monitor not found: {monitor_id}"}
 
-        scraper = PryScraper()
+            previous_content = monitor.last_content or ""
-        result = await scraper.scrape(monitor["target_url"])
-        current_content = result.get("content", "")
-    except (json.JSONDecodeError, ValueError) as e:
-        logger.exception("monitor_scrape_failed", extra={"monitor_id": monitor_id})
-        return {"error": f"Scrape failed: {e!s}"}
 
-    judger = ChangeJudger(use_llm=monitor.get("use_llm_judge", False))
+            try:
-    judgment = await judger.judge(previous_content, current_content, monitor.get("goal", ""))
+                from scraper import PryScraper
 
-    monitor["last_run_at"] = datetime.now(UTC).isoformat()
+                scraper = PryScraper()
-    monitor["total_checks"] += 1
+                result = await scraper.scrape(monitor.url)
-    if previous_content and current_content != previous_content:
+                current_content = result.get("content", "")
-        monitor["total_changes"] += 1
+            except Exception as e:
-        monitor["current_version"] += 1
+                logger.exception("monitor_scrape_failed", extra={"monitor_id": monitor_id})
-        snap = {
+                run = MonitorRun(
-            "version": monitor["current_version"],
+                    monitor_id=monitor_id,
-            "content": current_content,
+                    ts=datetime.now(UTC),
-            "detected_at": monitor["last_run_at"],
+                    status="error",
-        }
+                    error=str(e),
-        _snapshot_path(monitor_id, monitor["current_version"]).write_text(json.dumps(snap))
+                )
-    monitor["last_content"] = current_content
+                s.add(run)
-    path.write_text(json.dumps(monitor, indent=2))
+                return {"error": f"Scrape failed: {e!s}"}
 
-    result = {
+            judger = ChangeJudger(use_llm=monitor.use_llm_judge)
-        "monitor_id": monitor_id,
+            judgment = await judger.judge(previous_content, current_content, monitor.goal or "")
-        "name": monitor["name"],
-        "changed": bool(previous_content) and current_content != previous_content,
-        "previous_content_length": len(previous_content),
-        "current_content_length": len(current_content),
-        "judgment": judgment,
-        "total_checks": monitor["total_checks"],
-        "total_changes": monitor["total_changes"],
-    }
 
-    if judgment["meaningful"] and monitor.get("webhook_url"):
+            changed = bool(previous_content) and current_content != previous_content
-        await _fire_webhook(monitor["webhook_url"], result)
+            now = datetime.now(UTC)
+            monitor.last_run_at = now
+            monitor.total_checks = (monitor.total_checks or 0) + 1
 
-    return result
+            run_status = "success"
+            diff_data = {}
+            if changed:
+                monitor.total_changes = (monitor.total_changes or 0) + 1
+                monitor.current_version = (monitor.current_version or 0) + 1
+                run_status = "changed"
+                diff_data = {
+                    "previous_length": len(previous_content),
+                    "current_length": len(current_content),
+                    "diff": _compute_diff(previous_content, current_content),
+                }
+
+            monitor.last_content = current_content
+
+            run = MonitorRun(
+                monitor_id=monitor_id,
+                ts=now,
+                status=run_status,
+                diff=diff_data,
+            )
+            s.add(run)
+
+            result = {
+                "monitor_id": monitor_id,
+                "name": monitor.monitor_name,
+                "changed": changed,
+                "previous_content_length": len(previous_content),
+                "current_content_length": len(current_content),
+                "judgment": judgment,
+                "total_checks": monitor.total_checks,
+                "total_changes": monitor.total_changes,
+            }
+
+            webhook_url = monitor.webhook_url
+            meaningful = judgment["meaningful"]
+
+        if meaningful and webhook_url:
+            await _fire_webhook(webhook_url, result)
+
+        return result
+    except Exception:
+        logger.exception("monitor_run_failed", extra={"monitor_id": monitor_id})
+        return {"error": f"Monitor run failed: {monitor_id}"}
 
 
 async def _fire_webhook(webhook_url: str, payload: dict[str, Any]) -> None:
@@ -289,22 +320,29 @@ async def _fire_webhook(webhook_url: str, payload: dict[str, Any]) -> None:
 
 
 async def list_monitors() -> list[dict[str, Any]]:
-    monitors = []
+    try:
-    for path in sorted(MONITORS_DIR.glob("*.json"), key=os.path.getmtime, reverse=True):
+        with session_scope() as s:
-        try:
+            monitors = (
-            data = json.loads(path.read_text())
+                s.query(Monitor)
-            monitors.append(data)
+                .filter(Monitor.active.is_(True))
-        except (json.JSONDecodeError, OSError):
+                .order_by(Monitor.created_at.desc())
-            continue
+                .all()
-    return monitors
+            )
+            return [_monitor_to_dict(m) for m in monitors]
+    except Exception:
+        logger.exception("monitor_list_failed")
+        return []
 
 
 async def delete_monitor(monitor_id: str) -> bool:
-    path = _monitor_path(monitor_id)
+    try:
-    snap_dir = MONITORS_DIR / monitor_id
+        with session_scope() as s:
-    if snap_dir.exists():
+            monitor = s.query(Monitor).filter(Monitor.monitor_id == monitor_id).first()
-        shutil.rmtree(snap_dir)
+            if not monitor:
-    if path.exists():
+                return False
-        path.unlink()
+            s.delete(monitor)
+        logger.info("monitor_deleted", extra={"monitor_id": monitor_id})
         return True
-    return False
+    except Exception:
+        logger.exception("monitor_delete_failed", extra={"monitor_id": monitor_id})
+        return False

referrals.py

@@ -7,19 +7,18 @@ Supports x402 (HTTP 402) pay-per-scrape protocol for monetization."""
 #
 # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
 # Licensed under MIT. See LICENSE.
-import json
 import logging
 import uuid
 from datetime import UTC, datetime, timedelta
 from typing import Any
 
-from paths import PRY_DATA_DIR
+from sqlalchemy import func
+from sqlalchemy.exc import SQLAlchemyError
+
+from db import ReferralClick, ReferralConversion, ReferralInAppUsage, session_scope
 
 logger = logging.getLogger(__name__)
 
-REFERRAL_DIR = PRY_DATA_DIR / "referrals"
-REFERRAL_DIR.mkdir(parents=True, exist_ok=True)
-
 # Provider catalog with referral program details
 # Format: { category: [ {name, url, commission, cookie_days, ...} ] }
 PROVIDER_CATALOG: dict[str, list[dict[str, Any]]] = {
@@ -134,7 +133,7 @@ PROVIDER_CATALOG: dict[str, list[dict[str, Any]]] = {
         {
             "name": "Hetzner",
             "url": "https://hetzner.com/?ref=pry",
-            "commission": "€25/signup + 10% recurring",
+            "commission": "\u20ac25/signup + 10% recurring",
             "cookie_days": 90,
             "tag": "hetzner",
         },
@@ -176,7 +175,7 @@ PROVIDER_CATALOG: dict[str, list[dict[str, Any]]] = {
         {
             "name": "Netcup",
             "url": "https://netcup.com/?ref=pry",
-            "commission": "€5/signup",
+            "commission": "\u20ac5/signup",
             "cookie_days": 90,
             "tag": "netcup",
         },
@@ -588,71 +587,68 @@ PROVIDER_CATALOG: dict[str, list[dict[str, Any]]] = {
 class ReferralTracker:
     """Track referral clicks, conversions, and revenue."""
 
-    def __init__(self):
+    def __init__(self) -> None:
-        self.clicks_file = REFERRAL_DIR / "clicks.jsonl"
+        pass
-        self.conversions_file = REFERRAL_DIR / "conversions.jsonl"
-        self._load_state()
-
-    def _load_state(self) -> None:
-        self.clicks: list[dict[str, Any]] = []
-        self.conversions: list[dict[str, Any]] = []
-        for path in (self.clicks_file, self.conversions_file):
-            if path.exists():
-                try:
-                    for line in path.read_text().splitlines():
-                        if line.strip():
-                            record = json.loads(line)
-                            (self.clicks if path == self.clicks_file else self.conversions).append(
-                                record
-                            )
-                except (json.JSONDecodeError, OSError):
-                    pass
 
     def record_click(
         self, provider_tag: str, url: str, source: str = "pry", user_id: str = ""
     ) -> str:
         """Record a referral link click. Returns the tracking ID."""
         click_id = uuid.uuid4().hex[:12]
-        record = {
-            "id": click_id,
-            "provider": provider_tag,
-            "url": url,
-            "source": source,
-            "user_id": user_id,
-            "timestamp": datetime.now(UTC).isoformat(),
-            "converted": False,
-        }
         try:
-            with open(self.clicks_file, "a") as f:
+            with session_scope() as s:
-                f.write(json.dumps(record) + "\n")
+                s.add(
-            self.clicks.append(record)
+                    ReferralClick(
-        except OSError:
+                        click_id=click_id,
-            pass
+                        provider=provider_tag,
+                        url=url,
+                        source=source,
+                        user_id=user_id,
+                        converted=False,
+                        conversion_value_usd=0.0,
+                    )
+                )
+        except SQLAlchemyError as e:
+            logger.error(
+                "referral_click_failed",
+                extra={"click_id": click_id, "provider": provider_tag, "error": str(e)[:200]},
+            )
+            raise
         logger.info("referral_click", extra={"provider": provider_tag, "source": source})
         return click_id
 
     def record_conversion(self, click_id: str, revenue_usd: float = 0.0, notes: str = "") -> bool:
-        """Record a conversion for a click."""
+        """Record a conversion for a click. Returns True if successful."""
-        for click in self.clicks:
+        try:
-            if click["id"] == click_id:
+            with session_scope() as s:
-                click["converted"] = True
+                click = s.query(ReferralClick).filter_by(click_id=click_id).first()
-                click["revenue_usd"] = revenue_usd
+                if click is None:
-                click["conversion_notes"] = notes
+                    logger.warning(
-                conversion = {
+                        "referral_conversion_click_not_found",
-                    "click_id": click_id,
+                        extra={"click_id": click_id},
-                    "provider": click["provider"],
+                    )
-                    "revenue_usd": revenue_usd,
+                    return False
-                    "notes": notes,
+                click.converted = True
-                    "timestamp": datetime.now(UTC).isoformat(),
+                click.conversion_value_usd = revenue_usd
-                }
+                s.add(
-                try:
+                    ReferralConversion(
-                    with open(self.conversions_file, "a") as f:
+                        click_id=click_id,
-                        f.write(json.dumps(conversion) + "\n")
+                        provider=click.provider,
-                    self.conversions.append(conversion)
+                        revenue_usd=revenue_usd,
-                except OSError:
+                        notes=notes,
-                    pass
+                    )
-                return True
+                )
-        return False
+        except SQLAlchemyError as e:
+            logger.error(
+                "referral_conversion_failed",
+                extra={"click_id": click_id, "error": str(e)[:200]},
+            )
+            raise
+        logger.info(
+            "referral_conversion",
+            extra={"click_id": click_id, "revenue_usd": revenue_usd},
+        )
+        return True
 
     def get_catalog(self, category: str = "") -> list[dict[str, Any]]:
         """Get the provider catalog, optionally filtered by category."""
@@ -665,53 +661,77 @@ class ReferralTracker:
     def get_stats(self, days_back: int = 30) -> dict[str, Any]:
         """Get referral statistics for the last N days."""
         cutoff = datetime.now(UTC) - timedelta(days=days_back)
-        recent_clicks = []
+        try:
-        for c in self.clicks:
+            with session_scope() as s:
-            try:
+                total_clicks = (
-                ts = datetime.fromisoformat(c["timestamp"])
+                    s.query(func.count(ReferralClick.id))
-                if ts >= cutoff:
+                    .filter(ReferralClick.ts >= cutoff)
-                    recent_clicks.append(c)
+                    .scalar()
-            except (ValueError, TypeError):
+                    or 0
-                pass
+                )
-        recent_conversions = []
+
-        for c in self.conversions:
+                click_rows = (
-            try:
+                    s.query(
-                ts = datetime.fromisoformat(c.get("timestamp", ""))
+                        ReferralClick.provider,
-                if ts >= cutoff:
+                        func.count(ReferralClick.id).label("cnt"),
-                    recent_conversions.append(c)
+                    )
-            except (ValueError, TypeError):
+                    .filter(ReferralClick.ts >= cutoff)
-                pass
+                    .group_by(ReferralClick.provider)
-        clicks_by_provider: dict[str, int] = {}
+                    .all()
-        for c in recent_clicks:
+                )
-            clicks_by_provider[c["provider"]] = clicks_by_provider.get(c["provider"], 0) + 1
+                clicks_by_provider = {r.provider: r.cnt for r in click_rows}
-        conversions_by_provider: dict[str, int] = {}
+
-        revenue_by_provider: dict[str, float] = {}
+                conv_rows = (
-        for c in recent_conversions:
+                    s.query(
-            p = c["provider"]
+                        ReferralConversion.provider,
-            conversions_by_provider[p] = conversions_by_provider.get(p, 0) + 1
+                        func.count(ReferralConversion.id).label("cnt"),
-            revenue_by_provider[p] = revenue_by_provider.get(p, 0.0) + c.get("revenue_usd", 0.0)
+                        func.sum(ReferralConversion.revenue_usd).label("rev"),
-        return {
+                    )
-            "period_days": days_back,
+                    .filter(ReferralConversion.ts >= cutoff)
-            "total_clicks": len(recent_clicks),
+                    .group_by(ReferralConversion.provider)
-            "total_conversions": len(recent_conversions),
+                    .all()
-            "clicks_by_provider": clicks_by_provider,
+                )
-            "conversions_by_provider": conversions_by_provider,
+                conversions_by_provider = {r.provider: r.cnt for r in conv_rows}
-            "revenue_by_provider_usd": revenue_by_provider,
+                revenue_by_provider = {r.provider: float(r.rev or 0.0) for r in conv_rows}
-            "total_revenue_usd": sum(revenue_by_provider.values()),
+
-            "conversion_rate": len(recent_conversions) / max(len(recent_clicks), 1),
+                total_conversions = sum(conversions_by_provider.values())
-            "catalog_size": sum(len(v) for v in PROVIDER_CATALOG.values()),
+                total_revenue = sum(revenue_by_provider.values())
-        }
+
+                return {
+                    "period_days": days_back,
+                    "total_clicks": total_clicks,
+                    "total_conversions": total_conversions,
+                    "clicks_by_provider": clicks_by_provider,
+                    "conversions_by_provider": conversions_by_provider,
+                    "revenue_by_provider_usd": revenue_by_provider,
+                    "total_revenue_usd": total_revenue,
+                    "conversion_rate": total_conversions / max(total_clicks, 1),
+                    "catalog_size": sum(len(v) for v in PROVIDER_CATALOG.values()),
+                }
+        except SQLAlchemyError as e:
+            logger.error(
+                "referral_stats_failed",
+                extra={"error": str(e)[:200]},
+            )
+            raise
 
     def track_in_app_usage(self, provider_tag: str, revenue_usd: float) -> None:
         """Track revenue from in-app LLM provider usage (per-call)."""
-        record = {
-            "type": "in_app_usage",
-            "provider": provider_tag,
-            "revenue_usd": revenue_usd,
-            "timestamp": datetime.now(UTC).isoformat(),
-        }
         try:
-            with open(REFERRAL_DIR / "in_app_usage.jsonl", "a") as f:
+            with session_scope() as s:
-                f.write(json.dumps(record) + "\n")
+                s.add(
-        except OSError:
+                    ReferralInAppUsage(
-            pass
+                        provider=provider_tag,
+                        revenue_usd=revenue_usd,
+                    )
+                )
+        except SQLAlchemyError as e:
+            logger.error(
+                "referral_in_app_usage_failed",
+                extra={"provider": provider_tag, "error": str(e)[:200]},
+            )
+            raise
+        logger.info(
+            "referral_in_app_usage",
+            extra={"provider": provider_tag, "revenue_usd": revenue_usd},
+        )

structure_monitor.py

@@ -7,9 +7,7 @@ Track CSS selector validity over time. Alert before scrapers break."""
 # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
 # Licensed under MIT. See LICENSE.
 import hashlib
-import json
 import logging
-from contextlib import suppress
 from datetime import UTC, datetime
 from typing import Any
 
@@ -17,13 +15,10 @@ import httpx
 from lxml import html as lxml_html
 
 from client import get_client
-from paths import PRY_DATA_DIR
+from db import StructureSnapshot, session_scope
 
 logger = logging.getLogger(__name__)
 
-STRUCTURE_DIR = PRY_DATA_DIR / "structure"
-STRUCTURE_DIR.mkdir(parents=True, exist_ok=True)
-
 
 async def check_selectors(
     url: str,
@@ -46,7 +41,6 @@ async def check_selectors(
         "failed_count": 0,
     }
 
-    # Fetch the page
     try:
         client = await get_client()
         resp = await client.get(
@@ -65,10 +59,9 @@ async def check_selectors(
 
     try:
         tree = lxml_html.fromstring(html_content)
-    except (httpx.HTTPError, httpx.RequestError):
+    except Exception:  # noqa: BLE001
         return {"url": url, "error": "Failed to parse HTML", "all_matched": False}
 
-    # Check each selector
     for sel in selectors:
         selector_str = sel.get("selector", "")
         name = sel.get("name", selector_str[:30])
@@ -94,7 +87,6 @@ async def check_selectors(
                 "status": "ok" if is_matched else "broken",
             }
 
-            # If matched, sample content for change detection
             if is_matched and elements:
                 sample_text = " ".join(e.text_content().strip()[:100] for e in elements[:3])
                 selector_result["sample"] = sample_text[:200]
@@ -105,7 +97,7 @@ async def check_selectors(
             else:
                 result["failed_count"] += 1
 
-        except (httpx.HTTPError, httpx.RequestError) as e:
+        except Exception as e:  # noqa: BLE001
             result["selectors"].append(
                 {
                     "name": name,
@@ -133,80 +125,112 @@ async def monitor_page_structure(
     Returns the current check result plus any detected changes since last check.
     """
     url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
-    # Use template_id if provided for the filename, else url_hash
-    file_key = template_id or url_hash
-    history_path = STRUCTURE_DIR / f"structure_{file_key}.json"
 
-    # Run current check
     current = await check_selectors(url, selectors)
     if "error" in current:
         return current
 
-    # Load previous state
-    previous: dict[str, Any] | None = None
-    if history_path.exists():
-        with suppress(json.JSONDecodeError, OSError):
-            previous = json.loads(history_path.read_text())
-
-    # Detect changes
     changes = []
-    if previous and "selectors" in previous:
-        prev_map = {s["name"]: s for s in previous["selectors"]}
-        curr_map = {s["name"]: s for s in current["selectors"]}
 
-        for name, curr_sel in curr_map.items():
+    try:
-            prev_sel = prev_map.get(name)
+        with session_scope() as s:
-            if prev_sel:
+            if template_id:
-                # Check status change
+                previous_snap = (
-                if prev_sel.get("matched") and not curr_sel.get("matched"):
+                    s.query(StructureSnapshot)
-                    changes.append(
+                    .filter(StructureSnapshot.template_id == template_id)
-                        {
+                    .order_by(StructureSnapshot.checked_at.desc())
-                            "type": "selector_broken",
+                    .first()
-                            "severity": "critical",
+                )
-                            "name": name,
+            else:
-                            "selector": curr_sel.get("selector", ""),
+                previous_snap = (
-                            "message": f"CSS selector '{name}' stopped matching (was matching before)",
+                    s.query(StructureSnapshot)
-                        }
+                    .filter(StructureSnapshot.url_hash == url_hash)
-                    )
+                    .order_by(StructureSnapshot.checked_at.desc())
-                elif not prev_sel.get("matched") and curr_sel.get("matched"):
+                    .first()
-                    changes.append(
+                )
-                        {
-                            "type": "selector_fixed",
-                            "severity": "info",
-                            "name": name,
-                            "message": f"CSS selector '{name}' started matching again",
-                        }
-                    )
-                # Check count change
-                elif curr_sel.get("match_count") != prev_sel.get("match_count"):
-                    changes.append(
-                        {
-                            "type": "count_changed",
-                            "severity": "medium",
-                            "name": name,
-                            "message": f"Selector '{name}' matched {prev_sel.get('match_count')} elements before, now {curr_sel.get('match_count')}",
-                            "previous_count": prev_sel.get("match_count"),
-                            "current_count": curr_sel.get("match_count"),
-                        }
-                    )
 
-    # Detect new failures
+            if previous_snap:
-    if previous and previous.get("all_matched") and not current.get("all_matched"):
+                previous_selectors = previous_snap.selectors or []
-        changes.append(
+                previous_all_matched = previous_snap.all_matched
-            {
+            else:
-                "type": "structure_broken",
+                previous_selectors = []
-                "severity": "critical",
+                previous_all_matched = True
-                "name": "page_structure",
-                "message": f"Page structure changed — {current.get('failed_count')} selectors now fail",
-                "failed_selectors": [
-                    s["name"] for s in current.get("selectors", []) if not s.get("matched")
-                ],
-            }
-        )
 
-    # Save current state
+            prev_map = {s["name"]: s for s in previous_selectors if isinstance(s, dict)}
-    with suppress(OSError):
+            curr_map = {s["name"]: s for s in current.get("selectors", []) if isinstance(s, dict)}
-        history_path.write_text(json.dumps(current, indent=2))
+
+            for name, curr_sel in curr_map.items():
+                prev_sel = prev_map.get(name)
+                if prev_sel:
+                    if prev_sel.get("matched") and not curr_sel.get("matched"):
+                        changes.append(
+                            {
+                                "type": "selector_broken",
+                                "severity": "critical",
+                                "name": name,
+                                "selector": curr_sel.get("selector", ""),
+                                "message": f"CSS selector '{name}' stopped matching (was matching before)",
+                            }
+                        )
+                    elif not prev_sel.get("matched") and curr_sel.get("matched"):
+                        changes.append(
+                            {
+                                "type": "selector_fixed",
+                                "severity": "info",
+                                "name": name,
+                                "message": f"CSS selector '{name}' started matching again",
+                            }
+                        )
+                    elif curr_sel.get("match_count") != prev_sel.get("match_count"):
+                        changes.append(
+                            {
+                                "type": "count_changed",
+                                "severity": "medium",
+                                "name": name,
+                                "message": f"Selector '{name}' matched {prev_sel.get('match_count')} elements before, now {curr_sel.get('match_count')}",
+                                "previous_count": prev_sel.get("match_count"),
+                                "current_count": curr_sel.get("match_count"),
+                            }
+                        )
+
+            if previous_selectors and previous_all_matched and not current.get("all_matched"):
+                changes.append(
+                    {
+                        "type": "structure_broken",
+                        "severity": "critical",
+                        "name": "page_structure",
+                        "message": f"Page structure changed — {current.get('failed_count')} selectors now fail",
+                        "failed_selectors": [
+                            s["name"] for s in current.get("selectors", []) if not s.get("matched")
+                        ],
+                    }
+                )
+
+            now = datetime.now(UTC)
+            snapshot = StructureSnapshot(
+                url=url,
+                url_hash=url_hash,
+                template_id=template_id,
+                selectors=current.get("selectors", []),
+                all_matched=current.get("all_matched", False),
+                matched_count=current.get("matched_count", 0),
+                failed_count=current.get("failed_count", 0),
+                changes=changes,
+                change_count=len(changes),
+                has_changes=len(changes) > 0,
+                dom_summary={},
+                checked_at=now,
+            )
+            s.add(snapshot)
+
+    except Exception:
+        logger.exception("structure_monitor_save_failed", extra={"url": url})
+        current["changes"] = changes
+        current["change_count"] = len(changes)
+        current["has_changes"] = len(changes) > 0
+        current["template_id"] = template_id
+        current["url_hash"] = url_hash
+        return current
 
     current["changes"] = changes
     current["change_count"] = len(changes)
@@ -217,23 +241,36 @@ async def monitor_page_structure(
     return current
 
 
-def get_structure_history(url: str, template_id: str = "") -> dict[str, Any]:
+async def get_structure_history(url: str, template_id: str = "") -> dict[str, Any]:
     """Get structure check history for a URL."""
-    url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
-    file_key = template_id or url_hash
-    history_path = STRUCTURE_DIR / f"structure_{file_key}.json"
-
-    if not history_path.exists():
-        return {"url": url, "has_history": False}
-
     try:
-        data = json.loads(history_path.read_text())
+        with session_scope() as s:
-        return {
+            if template_id:
-            "url": url,
+                snap = (
-            "has_history": True,
+                    s.query(StructureSnapshot)
-            "last_checked": data.get("checked_at", ""),
+                    .filter(StructureSnapshot.template_id == template_id)
-            "last_all_matched": data.get("all_matched", True),
+                    .order_by(StructureSnapshot.checked_at.desc())
-            "last_selectors": data.get("selectors", []),
+                    .first()
-        }
+                )
-    except (json.JSONDecodeError, OSError):
+            else:
+                url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
+                snap = (
+                    s.query(StructureSnapshot)
+                    .filter(StructureSnapshot.url_hash == url_hash)
+                    .order_by(StructureSnapshot.checked_at.desc())
+                    .first()
+                )
+
+            if not snap:
+                return {"url": url, "has_history": False}
+
+            return {
+                "url": url,
+                "has_history": True,
+                "last_checked": snap.checked_at.isoformat() if snap.checked_at else "",
+                "last_all_matched": snap.all_matched,
+                "last_selectors": snap.selectors or [],
+            }
+    except Exception:
+        logger.exception("structure_history_fetch_failed", extra={"url": url})
         return {"url": url, "has_history": False}

tests/test_intelligence.py

@@ -5,8 +5,15 @@
 # Licensed under MIT. See LICENSE.
 """Tests for competitive intelligence engine."""
 
+import time
+from collections.abc import Iterator
+
+import pytest
+
+from db import Base, get_engine
 from intelligence import (
     compute_field_statistics,
+    detect_anomalies_numeric,
     generate_alert,
     generate_weekly_report,
     get_snapshots,
@@ -14,12 +21,52 @@ from intelligence import (
 )
 
 
+@pytest.fixture(autouse=True)
+def _intel_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    """Use in-memory SQLite for each test."""
+    monkeypatch.setenv("PRY_DATABASE_URL", "sqlite://")
+    import db as db_module
+
+    db_module._engine = None
+    db_module._SessionLocal = None
+    engine = get_engine()
+    Base.metadata.create_all(engine)
+    yield
+    Base.metadata.drop_all(engine)
+
+
+# -- Snapshot recording --
+
+
 def test_record_snapshot() -> None:
     result = record_snapshot(
-        "comp_1", "Competitor A", "https://example.com", {"price": 99.99, "stock": "in_stock"}
+        "comp_1",
+        "Competitor A",
+        "https://example.com",
+        {"price": 99.99, "stock": "in_stock"},
     )
     assert "ts" in result
+    assert "unix_ts" in result
     assert result["competitor_name"] == "Competitor A"
+    assert result["competitor_id"] == "comp_1"
+    assert result["fields"]["price"] == 99.99
+
+
+def test_record_snapshot_returns_iso_ts() -> None:
+    result = record_snapshot("comp_ts", "TS Co", "https://example.com", {"x": 1})
+    ts = result["ts"]
+    assert isinstance(ts, str)
+    assert ts.endswith("Z") or "+" in ts
+
+
+def test_record_snapshot_unix_ts() -> None:
+    before = time.time()
+    result = record_snapshot("comp_ut", "UT Co", "https://example.com", {"x": 1})
+    after = time.time()
+    assert before <= result["unix_ts"] <= after + 1
+
+
+# -- Retreiving snapshots --
 
 
 def test_get_snapshots() -> None:
@@ -27,39 +74,179 @@ def test_get_snapshots() -> None:
     snapshots = get_snapshots("comp_test")
     assert len(snapshots) >= 1
     assert snapshots[0]["competitor_name"] == "Test"
+    assert snapshots[0]["fields"]["a"] == 1
 
 
-def test_compute_field_statistics_numeric() -> None:
+def test_get_snapshots_limit() -> None:
-    snapshots = [
+    for i in range(10):
-        {"fields": {"price": 100}},
+        record_snapshot("comp_limit", "Limit Co", "https://example.com", {"i": i})
-        {"fields": {"price": 110}},
+    snapshots = get_snapshots("comp_limit", limit=3)
-        {"fields": {"price": 105}},
+    assert len(snapshots) == 3
-    ]
-    stats = compute_field_statistics(snapshots, "price")
-    assert stats["has_history"] is True
-    assert stats["mean"] == 105.0
-    assert stats["min"] == 100
-    assert stats["max"] == 110
 
 
-def test_generate_alert_price_change() -> None:
+def test_get_snapshots_ordered_by_ts_desc() -> None:
-    alert = generate_alert("Competitor A", "price", 100, 120)
+    for i in range(5):
-    assert "Competitor A" in alert
+        record_snapshot("comp_order", "Order Co", "https://example.com", {"i": i})
-    assert "increased" in alert or "changed" in alert
+        time.sleep(0.01)
+    snapshots = get_snapshots("comp_order")
+    assert len(snapshots) == 5
+    assert snapshots[0]["fields"]["i"] == 4
 
 
-def test_generate_alert_decrease() -> None:
+def test_get_snapshots_since_hours() -> None:
-    alert = generate_alert("Competitor B", "price", 200, 150)
+    record_snapshot("comp_since", "Since Co", "https://example.com", {"val": 1})
-    assert "decreased" in alert
+    snapshots = get_snapshots("comp_since", since_hours=1)
+    assert len(snapshots) >= 1
 
 
-def test_generate_alert_text_change() -> None:
+def test_get_snapshots_empty_for_unknown() -> None:
-    alert = generate_alert("Competitor C", "description", "Old desc", "New and improved")
+    snapshots = get_snapshots("nonexistent_comp")
-    assert "Competitor C" in alert
+    assert snapshots == []
 
 
-def test_generate_report() -> None:
+# -- Field statistics --
-    record_snapshot("comp_rpt", "Report Co", "https://example.com", {"price": 50})
+
-    report = generate_weekly_report([{"id": "comp_rpt", "name": "Report Co"}], days_back=7)
+
-    assert "report_period" in report
+class TestComputeFieldStatistics:
-    assert "generated_at" in report
+    def test_numeric(self) -> None:
+        snapshots = [
+            {"fields": {"price": 100}},
+            {"fields": {"price": 110}},
+            {"fields": {"price": 105}},
+        ]
+        stats = compute_field_statistics(snapshots, "price")
+        assert stats["has_history"] is True
+        assert stats["mean"] == 105.0
+        assert stats["min"] == 100
+        assert stats["max"] == 110
+
+    def test_insufficient_data(self) -> None:
+        snapshots = [{"fields": {"price": 100}}]
+        stats = compute_field_statistics(snapshots, "price")
+        assert stats["has_history"] is False
+
+    def test_empty_snapshots(self) -> None:
+        stats = compute_field_statistics([], "price")
+        assert stats["count"] == 0
+        assert stats["has_history"] is False
+
+    def test_missing_field(self) -> None:
+        snapshots = [{"fields": {"price": 100}}, {"fields": {"other": 200}}]
+        stats = compute_field_statistics(snapshots, "price")
+        assert stats["count"] == 1
+        assert stats["has_history"] is False
+
+    def test_string_field(self) -> None:
+        snapshots = [
+            {"fields": {"status": "active"}},
+            {"fields": {"status": "active"}},
+            {"fields": {"status": "paused"}},
+        ]
+        stats = compute_field_statistics(snapshots, "status")
+        assert stats["has_history"] is True
+        assert stats["unique_values"] == 2
+        assert stats["latest"] == "paused"
+
+
+# -- Anomaly detection --
+
+
+class TestDetectAnomaliesNumeric:
+    def test_detects_anomaly(self) -> None:
+        history = [100, 102, 101, 103, 99, 101, 102]
+        result = detect_anomalies_numeric(150, history)
+        assert result["anomaly"] is True
+        assert result["z_score"] >= 2.0
+
+    def test_insufficient_history(self) -> None:
+        result = detect_anomalies_numeric(100, [])
+        assert result["anomaly"] is False
+        assert "Insufficient history" in result["reason"]
+
+    def test_no_anomaly(self) -> None:
+        history = [100, 101, 100, 101, 100]
+        result = detect_anomalies_numeric(100, history)
+        assert result["anomaly"] is False
+
+    def test_constant_history_same_value(self) -> None:
+        history = [100, 100, 100]
+        result = detect_anomalies_numeric(100, history)
+        assert result["anomaly"] is False
+
+    def test_constant_history_diff_value(self) -> None:
+        history = [100, 100, 100]
+        result = detect_anomalies_numeric(200, history)
+        assert result["anomaly"] is True
+        assert "constant history" in result["reason"]
+
+
+# -- Alerts --
+
+
+class TestGenerateAlert:
+    def test_price_increase(self) -> None:
+        alert = generate_alert("Competitor A", "price", 100, 120)
+        assert "Competitor A" in alert
+        assert "increased" in alert
+
+    def test_price_decrease(self) -> None:
+        alert = generate_alert("Competitor B", "price", 200, 150)
+        assert "decreased" in alert
+
+    def test_text_change_short(self) -> None:
+        alert = generate_alert("Competitor C", "description", "Old desc", "New and improved")
+        assert "Competitor C" in alert
+        assert '"Old desc"' in alert
+
+    def test_text_change_long(self) -> None:
+        long_old = "x" * 60
+        long_new = "y" * 60
+        alert = generate_alert("Competitor D", "description", long_old, long_new)
+        assert "chars" in alert
+
+    def test_with_anomaly_info(self) -> None:
+        alert = generate_alert(
+            "Competitor F",
+            "price",
+            100,
+            200,
+            anomaly_info={"anomaly": True, "z_score": 3.5, "pct_change": 100.0},
+        )
+        assert "ANOMALY DETECTED" in alert
+        assert "z-score" in alert
+
+
+# -- Weekly report --
+
+
+class TestGenerateWeeklyReport:
+    def test_basic_report(self) -> None:
+        record_snapshot("comp_rpt", "Report Co", "https://example.com", {"price": 50})
+        report = generate_weekly_report([{"id": "comp_rpt", "name": "Report Co"}], days_back=7)
+        assert "report_period" in report
+        assert "generated_at" in report
+        assert report["competitors_tracked"] == 1
+
+    def test_report_with_changes(self) -> None:
+        record_snapshot("comp_chg", "Chg Co", "https://example.com", {"price": 100})
+        record_snapshot("comp_chg", "Chg Co", "https://example.com", {"price": 150})
+        report = generate_weekly_report([{"id": "comp_chg", "name": "Chg Co"}], days_back=7)
+        assert report["total_changes"] >= 1
+        assert report["most_active_competitor"] == "Chg Co"
+
+    def test_report_no_activity(self) -> None:
+        report = generate_weekly_report(
+            [{"id": "comp_inactive", "name": "Inactive Co"}],
+            days_back=7,
+        )
+        assert report["total_changes"] == 0
+
+    def test_report_multiple_competitors(self) -> None:
+        record_snapshot("comp_a", "Alpha Co", "https://example.com", {"price": 50})
+        record_snapshot("comp_b", "Beta Co", "https://example.com", {"price": 75})
+        competitors = [
+            {"id": "comp_a", "name": "Alpha Co"},
+            {"id": "comp_b", "name": "Beta Co"},
+        ]
+        report = generate_weekly_report(competitors, days_back=7)
+        assert report["competitors_tracked"] == 2

tests/test_monitor.py

@@ -5,7 +5,125 @@
 # Licensed under MIT. See LICENSE.
 """Tests for content monitors."""
 
-from monitor import ChangeJudger
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from monitor import ChangeJudger, create_monitor, delete_monitor, list_monitors, run_monitor
+
+
+@pytest.fixture
+def temp_db(monkeypatch, tmp_path):
+    import db as db_module
+
+    db_path = tmp_path / "test.db"
+    monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{db_path}")
+    monkeypatch.setenv("PRY_DATA_DIR", str(tmp_path))
+    db_module._engine = None
+    db_module._SessionLocal = None
+    db_module.get_engine()
+    yield
+    db_module._engine = None
+    db_module._SessionLocal = None
+
+
+@pytest.mark.asyncio
+async def test_create_monitor(temp_db):
+    result = await create_monitor("test-monitor", "https://example.com")
+    assert result["name"] == "test-monitor"
+    assert result["target_url"] == "https://example.com"
+    assert len(result["id"]) == 12
+    assert result["status"] == "active"
+
+
+@pytest.mark.asyncio
+async def test_create_monitor_with_all_options(temp_db):
+    result = await create_monitor(
+        "full-monitor",
+        "https://example.com",
+        schedule_cron="0 */12 * * *",
+        goal="find price changes",
+        webhook_url="https://hooks.example.com/notify",
+        use_llm_judge=True,
+    )
+    assert result["name"] == "full-monitor"
+    assert result["schedule_cron"] == "0 */12 * * *"
+    assert result["goal"] == "find price changes"
+    assert result["webhook_url"] == "https://hooks.example.com/notify"
+    assert result["use_llm_judge"] is True
+
+
+@pytest.mark.asyncio
+async def test_list_monitors(temp_db):
+    await create_monitor("m1", "https://example.com")
+    await create_monitor("m2", "https://example.org")
+    monitors = await list_monitors()
+    assert len(monitors) == 2
+
+
+@pytest.mark.asyncio
+async def test_list_monitors_empty(temp_db):
+    monitors = await list_monitors()
+    assert monitors == []
+
+
+@pytest.mark.asyncio
+async def test_delete_monitor(temp_db):
+    m = await create_monitor("to-delete", "https://example.com")
+    assert await delete_monitor(m["id"]) is True
+    monitors = await list_monitors()
+    assert len(monitors) == 0
+
+
+@pytest.mark.asyncio
+async def test_delete_nonexistent_monitor(temp_db):
+    assert await delete_monitor("nonexistent") is False
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_with_change(temp_db):
+    m = await create_monitor("change-test", "https://example.com")
+
+    mock_scraper = AsyncMock()
+    mock_scraper.scrape.return_value = {"content": "new content here"}
+
+    with patch("scraper.PryScraper", return_value=mock_scraper):
+        result = await run_monitor(m["id"])
+
+    assert result["monitor_id"] == m["id"]
+    assert result["current_content_length"] > 0
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_no_change(temp_db):
+    m = await create_monitor("no-change", "https://example.com")
+
+    mock_scraper = AsyncMock()
+    mock_scraper.scrape.return_value = {"content": ""}
+
+    with patch("scraper.PryScraper", return_value=mock_scraper):
+        result = await run_monitor(m["id"])
+
+    assert result["changed"] is False
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_not_found(temp_db):
+    result = await run_monitor("nonexistent")
+    assert "error" in result
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_scrape_error(temp_db):
+    m = await create_monitor("scrape-error", "https://example.com")
+
+    mock_scraper = AsyncMock()
+    mock_scraper.scrape.side_effect = ValueError("connection refused")
+
+    with patch("scraper.PryScraper", return_value=mock_scraper):
+        result = await run_monitor(m["id"])
+
+    assert "error" in result
 
 
 async def _judge(previous: str, current: str, goal: str = "") -> dict:

tests/test_referrals.py

@@ -5,10 +5,28 @@
 # Licensed under MIT. See LICENSE.
 """Tests for referral/affiliate system."""
 
+from collections.abc import Iterator
+
+import pytest
+
 from referrals import PROVIDER_CATALOG, ReferralTracker
 from x402 import X402_PRICING, X402Handler
 
 
+@pytest.fixture(autouse=True)
+def _referral_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    """Use in-memory SQLite for all referral tests."""
+    monkeypatch.setenv("PRY_DATABASE_URL", "sqlite://")
+    import db as _db
+
+    _db._engine = None
+    _db._SessionLocal = None
+    _db.get_engine()
+    yield
+    _db._engine = None
+    _db._SessionLocal = None
+
+
 def test_catalog_completeness() -> None:
     """We have a good catalog of providers."""
     total = sum(len(v) for v in PROVIDER_CATALOG.values())
@@ -41,28 +59,102 @@ def test_provider_format() -> None:
 
 
 def test_referral_click() -> None:
+    """Record a click and verify it exists in the database."""
+    from db import ReferralClick, session_scope
+
     rt = ReferralTracker()
     click_id = rt.record_click(
         "openai", "https://platform.openai.com/signup?via=pry", source="test"
     )
     assert click_id is not None
-    assert any(c["id"] == click_id for c in rt.clicks)
+    with session_scope() as s:
+        click = s.query(ReferralClick).filter_by(click_id=click_id).first()
+        assert click is not None
+        assert click.provider == "openai"
+        assert click.source == "test"
+        assert click.converted is False
 
 
 def test_referral_conversion() -> None:
+    """Record a click, convert it, and verify both tables."""
+    from db import ReferralClick, ReferralConversion, session_scope
+
     rt = ReferralTracker()
     click_id = rt.record_click("anthropic", "https://console.anthropic.com/?ref=pry")
     success = rt.record_conversion(click_id, revenue_usd=50.0, notes="Annual plan")
     assert success is True
+    with session_scope() as s:
+        click = s.query(ReferralClick).filter_by(click_id=click_id).first()
+        assert click.converted is True
+        assert click.conversion_value_usd == 50.0
+        conv = s.query(ReferralConversion).filter_by(click_id=click_id).first()
+        assert conv is not None
+        assert conv.revenue_usd == 50.0
+        assert conv.notes == "Annual plan"
+
+
+def test_referral_missing_click_conversion() -> None:
+    """Converting a non-existent click returns False."""
+    rt = ReferralTracker()
+    success = rt.record_conversion("nonexistent_click_id")
+    assert success is False
 
 
 def test_referral_stats() -> None:
+    """Record a click + conversion and verify stats."""
     rt = ReferralTracker()
     click_id = rt.record_click("openai", "https://platform.openai.com/signup?via=pry")
     rt.record_conversion(click_id, revenue_usd=100.0)
     stats = rt.get_stats(days_back=1)
     assert stats["total_clicks"] >= 1
+    assert stats["total_conversions"] >= 1
     assert stats["total_revenue_usd"] >= 100.0
+    assert stats["clicks_by_provider"].get("openai", 0) >= 1
+    assert stats["conversions_by_provider"].get("openai", 0) >= 1
+    assert stats["revenue_by_provider_usd"].get("openai", 0.0) >= 100.0
+    assert 0 < stats["conversion_rate"] <= 1.0
+    assert stats["period_days"] == 1
+    assert stats["catalog_size"] >= 50
+
+
+def test_referral_stats_empty() -> None:
+    """Stats with no data returns zeroes."""
+    rt = ReferralTracker()
+    stats = rt.get_stats(days_back=1)
+    assert stats["total_clicks"] == 0
+    assert stats["total_conversions"] == 0
+    assert stats["total_revenue_usd"] == 0.0
+    assert stats["conversion_rate"] == 0.0
+
+
+def test_referral_in_app_usage() -> None:
+    """Track in-app usage revenue and verify in database."""
+    from db import ReferralInAppUsage, session_scope
+
+    rt = ReferralTracker()
+    rt.track_in_app_usage("openai", 15.0)
+    with session_scope() as s:
+        usage = s.query(ReferralInAppUsage).filter_by(provider="openai").first()
+        assert usage is not None
+        assert usage.revenue_usd == 15.0
+
+
+def test_referral_catalog() -> None:
+    """Verify get_catalog returns correct data."""
+    rt = ReferralTracker()
+    full = rt.get_catalog()
+    assert len(full) > 0
+    for entry in full:
+        assert "category" in entry
+        assert "providers" in entry
+    llm = rt.get_catalog("llm")
+    assert len(llm) > 0
+    assert llm[0]["name"] == "OpenAI"
+    empty = rt.get_catalog("nonexistent")
+    assert empty == []
+
+
+# ── x402 tests (unchanged) ────────────────────────────────────────
 
 
 def test_x402_payment_request() -> None:
@@ -71,7 +163,7 @@ def test_x402_payment_request() -> None:
     assert body["x402Version"] == 1
     assert "accepts" in body
     base_accept = next(a for a in body["accepts"] if a["network"] == "base")
-    assert base_accept["maxAmountRequired"] == "1000"  # $0.001 = 1000 USDC units
+    assert base_accept["maxAmountRequired"] == "1000"
     assert base_accept["asset"] == "USDC"
     assert "PAYMENT-REQUIRED" in headers
 

tests/test_structure_monitor.py

@@ -7,6 +7,8 @@
 
 from unittest.mock import AsyncMock, patch
 
+import pytest
+
 from structure_monitor import (
     check_selectors,
     get_structure_history,
@@ -14,26 +16,125 @@ from structure_monitor import (
 )
 
 
-def test_check_selectors_empty() -> None:
+@pytest.fixture
-    import asyncio
+def temp_db(monkeypatch, tmp_path):
+    import db as db_module
 
-    result = asyncio.run(check_selectors("https://example.com", []))
+    db_path = tmp_path / "test.db"
+    monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{db_path}")
+    monkeypatch.setenv("PRY_DATA_DIR", str(tmp_path))
+    db_module._engine = None
+    db_module._SessionLocal = None
+    db_module.get_engine()
+    yield
+    db_module._engine = None
+    db_module._SessionLocal = None
+
+
+@pytest.mark.asyncio
+async def test_check_selectors_empty():
+    result = await check_selectors("https://example.com", [])
     assert "selectors" in result
+    assert result["all_matched"] is True
 
 
-def test_get_structure_history_no_history() -> None:
+@pytest.mark.asyncio
-    result = get_structure_history("https://nonexistent-page-test-123.com")
+async def test_get_structure_history_no_history(temp_db):
+    result = await get_structure_history("https://nonexistent-page-test-123.com")
     assert result["has_history"] is False
 
 
-def test_monitor_no_selectors() -> None:
+@pytest.mark.asyncio
-    import asyncio
+async def test_monitor_page_structure_creates_history(temp_db):
-
     mock_client = AsyncMock()
     mock_client.get.return_value.is_success = True
-    mock_client.get.return_value.text = "<html><body>test</body></html>"
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
 
     with patch("structure_monitor.get_client", return_value=mock_client):
-        result = asyncio.run(monitor_page_structure("https://example.com", []))
+        result = await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
     assert "selectors" in result
     assert "changes" in result
+    assert result["all_matched"] is True
+    assert result["has_changes"] is False
+
+
+@pytest.mark.asyncio
+async def test_get_structure_history_with_data(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = True
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
+    result = await get_structure_history("https://example.com")
+    assert result["has_history"] is True
+    assert "last_checked" in result
+    assert "last_selectors" in result
+
+
+@pytest.mark.asyncio
+async def test_monitor_page_structure_with_template_id(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = True
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        result = await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+            template_id="template-1",
+        )
+
+    assert result["template_id"] == "template-1"
+
+    by_url = await get_structure_history("https://example.com")
+    assert by_url["has_history"] is True
+
+    by_template = await get_structure_history("https://example.com", template_id="template-1")
+    assert by_template["has_history"] is True
+
+
+@pytest.mark.asyncio
+async def test_monitor_page_structure_detects_change(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = True
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
+    mock_client.get.return_value.text = "<html><body><h1>New Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        result = await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h2", "type": "css"}],
+        )
+
+    assert "changes" in result
+
+
+@pytest.mark.asyncio
+async def test_structure_page_http_error(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = False
+    mock_client.get.return_value.status_code = 404
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        result = await monitor_page_structure(
+            "https://example.com/404",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
+    assert "error" in result