feat(sql): migrate intelligence, monitor, and referrals to SQLAlchemy (#4)

tests/test_intelligence.py

@@ -5,8 +5,15 @@
 # Licensed under MIT. See LICENSE.
 """Tests for competitive intelligence engine."""
 
+import time
+from collections.abc import Iterator
+
+import pytest
+
+from db import Base, get_engine
 from intelligence import (
     compute_field_statistics,
+    detect_anomalies_numeric,
     generate_alert,
     generate_weekly_report,
     get_snapshots,
@@ -14,12 +21,52 @@ from intelligence import (
 )
 
 
+@pytest.fixture(autouse=True)
+def _intel_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    """Use in-memory SQLite for each test."""
+    monkeypatch.setenv("PRY_DATABASE_URL", "sqlite://")
+    import db as db_module
+
+    db_module._engine = None
+    db_module._SessionLocal = None
+    engine = get_engine()
+    Base.metadata.create_all(engine)
+    yield
+    Base.metadata.drop_all(engine)
+
+
+# -- Snapshot recording --
+
+
 def test_record_snapshot() -> None:
     result = record_snapshot(
-        "comp_1", "Competitor A", "https://example.com", {"price": 99.99, "stock": "in_stock"}
+        "comp_1",
+        "Competitor A",
+        "https://example.com",
+        {"price": 99.99, "stock": "in_stock"},
     )
     assert "ts" in result
+    assert "unix_ts" in result
     assert result["competitor_name"] == "Competitor A"
+    assert result["competitor_id"] == "comp_1"
+    assert result["fields"]["price"] == 99.99
+
+
+def test_record_snapshot_returns_iso_ts() -> None:
+    result = record_snapshot("comp_ts", "TS Co", "https://example.com", {"x": 1})
+    ts = result["ts"]
+    assert isinstance(ts, str)
+    assert ts.endswith("Z") or "+" in ts
+
+
+def test_record_snapshot_unix_ts() -> None:
+    before = time.time()
+    result = record_snapshot("comp_ut", "UT Co", "https://example.com", {"x": 1})
+    after = time.time()
+    assert before <= result["unix_ts"] <= after + 1
+
+
+# -- Retreiving snapshots --
 
 
 def test_get_snapshots() -> None:
@@ -27,39 +74,179 @@ def test_get_snapshots() -> None:
     snapshots = get_snapshots("comp_test")
     assert len(snapshots) >= 1
     assert snapshots[0]["competitor_name"] == "Test"
+    assert snapshots[0]["fields"]["a"] == 1
 
 
-def test_compute_field_statistics_numeric() -> None:
-    snapshots = [
-        {"fields": {"price": 100}},
-        {"fields": {"price": 110}},
-        {"fields": {"price": 105}},
-    ]
-    stats = compute_field_statistics(snapshots, "price")
-    assert stats["has_history"] is True
-    assert stats["mean"] == 105.0
-    assert stats["min"] == 100
-    assert stats["max"] == 110
+def test_get_snapshots_limit() -> None:
+    for i in range(10):
+        record_snapshot("comp_limit", "Limit Co", "https://example.com", {"i": i})
+    snapshots = get_snapshots("comp_limit", limit=3)
+    assert len(snapshots) == 3
 
 
-def test_generate_alert_price_change() -> None:
-    alert = generate_alert("Competitor A", "price", 100, 120)
-    assert "Competitor A" in alert
-    assert "increased" in alert or "changed" in alert
+def test_get_snapshots_ordered_by_ts_desc() -> None:
+    for i in range(5):
+        record_snapshot("comp_order", "Order Co", "https://example.com", {"i": i})
+        time.sleep(0.01)
+    snapshots = get_snapshots("comp_order")
+    assert len(snapshots) == 5
+    assert snapshots[0]["fields"]["i"] == 4
 
 
-def test_generate_alert_decrease() -> None:
-    alert = generate_alert("Competitor B", "price", 200, 150)
-    assert "decreased" in alert
+def test_get_snapshots_since_hours() -> None:
+    record_snapshot("comp_since", "Since Co", "https://example.com", {"val": 1})
+    snapshots = get_snapshots("comp_since", since_hours=1)
+    assert len(snapshots) >= 1
 
 
-def test_generate_alert_text_change() -> None:
-    alert = generate_alert("Competitor C", "description", "Old desc", "New and improved")
-    assert "Competitor C" in alert
+def test_get_snapshots_empty_for_unknown() -> None:
+    snapshots = get_snapshots("nonexistent_comp")
+    assert snapshots == []
 
 
-def test_generate_report() -> None:
-    record_snapshot("comp_rpt", "Report Co", "https://example.com", {"price": 50})
-    report = generate_weekly_report([{"id": "comp_rpt", "name": "Report Co"}], days_back=7)
-    assert "report_period" in report
-    assert "generated_at" in report
+# -- Field statistics --
+
+
+class TestComputeFieldStatistics:
+    def test_numeric(self) -> None:
+        snapshots = [
+            {"fields": {"price": 100}},
+            {"fields": {"price": 110}},
+            {"fields": {"price": 105}},
+        ]
+        stats = compute_field_statistics(snapshots, "price")
+        assert stats["has_history"] is True
+        assert stats["mean"] == 105.0
+        assert stats["min"] == 100
+        assert stats["max"] == 110
+
+    def test_insufficient_data(self) -> None:
+        snapshots = [{"fields": {"price": 100}}]
+        stats = compute_field_statistics(snapshots, "price")
+        assert stats["has_history"] is False
+
+    def test_empty_snapshots(self) -> None:
+        stats = compute_field_statistics([], "price")
+        assert stats["count"] == 0
+        assert stats["has_history"] is False
+
+    def test_missing_field(self) -> None:
+        snapshots = [{"fields": {"price": 100}}, {"fields": {"other": 200}}]
+        stats = compute_field_statistics(snapshots, "price")
+        assert stats["count"] == 1
+        assert stats["has_history"] is False
+
+    def test_string_field(self) -> None:
+        snapshots = [
+            {"fields": {"status": "active"}},
+            {"fields": {"status": "active"}},
+            {"fields": {"status": "paused"}},
+        ]
+        stats = compute_field_statistics(snapshots, "status")
+        assert stats["has_history"] is True
+        assert stats["unique_values"] == 2
+        assert stats["latest"] == "paused"
+
+
+# -- Anomaly detection --
+
+
+class TestDetectAnomaliesNumeric:
+    def test_detects_anomaly(self) -> None:
+        history = [100, 102, 101, 103, 99, 101, 102]
+        result = detect_anomalies_numeric(150, history)
+        assert result["anomaly"] is True
+        assert result["z_score"] >= 2.0
+
+    def test_insufficient_history(self) -> None:
+        result = detect_anomalies_numeric(100, [])
+        assert result["anomaly"] is False
+        assert "Insufficient history" in result["reason"]
+
+    def test_no_anomaly(self) -> None:
+        history = [100, 101, 100, 101, 100]
+        result = detect_anomalies_numeric(100, history)
+        assert result["anomaly"] is False
+
+    def test_constant_history_same_value(self) -> None:
+        history = [100, 100, 100]
+        result = detect_anomalies_numeric(100, history)
+        assert result["anomaly"] is False
+
+    def test_constant_history_diff_value(self) -> None:
+        history = [100, 100, 100]
+        result = detect_anomalies_numeric(200, history)
+        assert result["anomaly"] is True
+        assert "constant history" in result["reason"]
+
+
+# -- Alerts --
+
+
+class TestGenerateAlert:
+    def test_price_increase(self) -> None:
+        alert = generate_alert("Competitor A", "price", 100, 120)
+        assert "Competitor A" in alert
+        assert "increased" in alert
+
+    def test_price_decrease(self) -> None:
+        alert = generate_alert("Competitor B", "price", 200, 150)
+        assert "decreased" in alert
+
+    def test_text_change_short(self) -> None:
+        alert = generate_alert("Competitor C", "description", "Old desc", "New and improved")
+        assert "Competitor C" in alert
+        assert '"Old desc"' in alert
+
+    def test_text_change_long(self) -> None:
+        long_old = "x" * 60
+        long_new = "y" * 60
+        alert = generate_alert("Competitor D", "description", long_old, long_new)
+        assert "chars" in alert
+
+    def test_with_anomaly_info(self) -> None:
+        alert = generate_alert(
+            "Competitor F",
+            "price",
+            100,
+            200,
+            anomaly_info={"anomaly": True, "z_score": 3.5, "pct_change": 100.0},
+        )
+        assert "ANOMALY DETECTED" in alert
+        assert "z-score" in alert
+
+
+# -- Weekly report --
+
+
+class TestGenerateWeeklyReport:
+    def test_basic_report(self) -> None:
+        record_snapshot("comp_rpt", "Report Co", "https://example.com", {"price": 50})
+        report = generate_weekly_report([{"id": "comp_rpt", "name": "Report Co"}], days_back=7)
+        assert "report_period" in report
+        assert "generated_at" in report
+        assert report["competitors_tracked"] == 1
+
+    def test_report_with_changes(self) -> None:
+        record_snapshot("comp_chg", "Chg Co", "https://example.com", {"price": 100})
+        record_snapshot("comp_chg", "Chg Co", "https://example.com", {"price": 150})
+        report = generate_weekly_report([{"id": "comp_chg", "name": "Chg Co"}], days_back=7)
+        assert report["total_changes"] >= 1
+        assert report["most_active_competitor"] == "Chg Co"
+
+    def test_report_no_activity(self) -> None:
+        report = generate_weekly_report(
+            [{"id": "comp_inactive", "name": "Inactive Co"}],
+            days_back=7,
+        )
+        assert report["total_changes"] == 0
+
+    def test_report_multiple_competitors(self) -> None:
+        record_snapshot("comp_a", "Alpha Co", "https://example.com", {"price": 50})
+        record_snapshot("comp_b", "Beta Co", "https://example.com", {"price": 75})
+        competitors = [
+            {"id": "comp_a", "name": "Alpha Co"},
+            {"id": "comp_b", "name": "Beta Co"},
+        ]
+        report = generate_weekly_report(competitors, days_back=7)
+        assert report["competitors_tracked"] == 2

tests/test_monitor.py

@@ -5,7 +5,125 @@
 # Licensed under MIT. See LICENSE.
 """Tests for content monitors."""
 
-from monitor import ChangeJudger
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from monitor import ChangeJudger, create_monitor, delete_monitor, list_monitors, run_monitor
+
+
+@pytest.fixture
+def temp_db(monkeypatch, tmp_path):
+    import db as db_module
+
+    db_path = tmp_path / "test.db"
+    monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{db_path}")
+    monkeypatch.setenv("PRY_DATA_DIR", str(tmp_path))
+    db_module._engine = None
+    db_module._SessionLocal = None
+    db_module.get_engine()
+    yield
+    db_module._engine = None
+    db_module._SessionLocal = None
+
+
+@pytest.mark.asyncio
+async def test_create_monitor(temp_db):
+    result = await create_monitor("test-monitor", "https://example.com")
+    assert result["name"] == "test-monitor"
+    assert result["target_url"] == "https://example.com"
+    assert len(result["id"]) == 12
+    assert result["status"] == "active"
+
+
+@pytest.mark.asyncio
+async def test_create_monitor_with_all_options(temp_db):
+    result = await create_monitor(
+        "full-monitor",
+        "https://example.com",
+        schedule_cron="0 */12 * * *",
+        goal="find price changes",
+        webhook_url="https://hooks.example.com/notify",
+        use_llm_judge=True,
+    )
+    assert result["name"] == "full-monitor"
+    assert result["schedule_cron"] == "0 */12 * * *"
+    assert result["goal"] == "find price changes"
+    assert result["webhook_url"] == "https://hooks.example.com/notify"
+    assert result["use_llm_judge"] is True
+
+
+@pytest.mark.asyncio
+async def test_list_monitors(temp_db):
+    await create_monitor("m1", "https://example.com")
+    await create_monitor("m2", "https://example.org")
+    monitors = await list_monitors()
+    assert len(monitors) == 2
+
+
+@pytest.mark.asyncio
+async def test_list_monitors_empty(temp_db):
+    monitors = await list_monitors()
+    assert monitors == []
+
+
+@pytest.mark.asyncio
+async def test_delete_monitor(temp_db):
+    m = await create_monitor("to-delete", "https://example.com")
+    assert await delete_monitor(m["id"]) is True
+    monitors = await list_monitors()
+    assert len(monitors) == 0
+
+
+@pytest.mark.asyncio
+async def test_delete_nonexistent_monitor(temp_db):
+    assert await delete_monitor("nonexistent") is False
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_with_change(temp_db):
+    m = await create_monitor("change-test", "https://example.com")
+
+    mock_scraper = AsyncMock()
+    mock_scraper.scrape.return_value = {"content": "new content here"}
+
+    with patch("scraper.PryScraper", return_value=mock_scraper):
+        result = await run_monitor(m["id"])
+
+    assert result["monitor_id"] == m["id"]
+    assert result["current_content_length"] > 0
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_no_change(temp_db):
+    m = await create_monitor("no-change", "https://example.com")
+
+    mock_scraper = AsyncMock()
+    mock_scraper.scrape.return_value = {"content": ""}
+
+    with patch("scraper.PryScraper", return_value=mock_scraper):
+        result = await run_monitor(m["id"])
+
+    assert result["changed"] is False
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_not_found(temp_db):
+    result = await run_monitor("nonexistent")
+    assert "error" in result
+
+
+@pytest.mark.asyncio
+async def test_run_monitor_scrape_error(temp_db):
+    m = await create_monitor("scrape-error", "https://example.com")
+
+    mock_scraper = AsyncMock()
+    mock_scraper.scrape.side_effect = ValueError("connection refused")
+
+    with patch("scraper.PryScraper", return_value=mock_scraper):
+        result = await run_monitor(m["id"])
+
+    assert "error" in result
 
 
 async def _judge(previous: str, current: str, goal: str = "") -> dict:

tests/test_referrals.py

@@ -5,10 +5,28 @@
 # Licensed under MIT. See LICENSE.
 """Tests for referral/affiliate system."""
 
+from collections.abc import Iterator
+
+import pytest
+
 from referrals import PROVIDER_CATALOG, ReferralTracker
 from x402 import X402_PRICING, X402Handler
 
 
+@pytest.fixture(autouse=True)
+def _referral_db(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    """Use in-memory SQLite for all referral tests."""
+    monkeypatch.setenv("PRY_DATABASE_URL", "sqlite://")
+    import db as _db
+
+    _db._engine = None
+    _db._SessionLocal = None
+    _db.get_engine()
+    yield
+    _db._engine = None
+    _db._SessionLocal = None
+
+
 def test_catalog_completeness() -> None:
     """We have a good catalog of providers."""
     total = sum(len(v) for v in PROVIDER_CATALOG.values())
@@ -41,28 +59,102 @@ def test_provider_format() -> None:
 
 
 def test_referral_click() -> None:
+    """Record a click and verify it exists in the database."""
+    from db import ReferralClick, session_scope
+
     rt = ReferralTracker()
     click_id = rt.record_click(
         "openai", "https://platform.openai.com/signup?via=pry", source="test"
     )
     assert click_id is not None
-    assert any(c["id"] == click_id for c in rt.clicks)
+    with session_scope() as s:
+        click = s.query(ReferralClick).filter_by(click_id=click_id).first()
+        assert click is not None
+        assert click.provider == "openai"
+        assert click.source == "test"
+        assert click.converted is False
 
 
 def test_referral_conversion() -> None:
+    """Record a click, convert it, and verify both tables."""
+    from db import ReferralClick, ReferralConversion, session_scope
+
     rt = ReferralTracker()
     click_id = rt.record_click("anthropic", "https://console.anthropic.com/?ref=pry")
     success = rt.record_conversion(click_id, revenue_usd=50.0, notes="Annual plan")
     assert success is True
+    with session_scope() as s:
+        click = s.query(ReferralClick).filter_by(click_id=click_id).first()
+        assert click.converted is True
+        assert click.conversion_value_usd == 50.0
+        conv = s.query(ReferralConversion).filter_by(click_id=click_id).first()
+        assert conv is not None
+        assert conv.revenue_usd == 50.0
+        assert conv.notes == "Annual plan"
+
+
+def test_referral_missing_click_conversion() -> None:
+    """Converting a non-existent click returns False."""
+    rt = ReferralTracker()
+    success = rt.record_conversion("nonexistent_click_id")
+    assert success is False
 
 
 def test_referral_stats() -> None:
+    """Record a click + conversion and verify stats."""
     rt = ReferralTracker()
     click_id = rt.record_click("openai", "https://platform.openai.com/signup?via=pry")
     rt.record_conversion(click_id, revenue_usd=100.0)
     stats = rt.get_stats(days_back=1)
     assert stats["total_clicks"] >= 1
+    assert stats["total_conversions"] >= 1
     assert stats["total_revenue_usd"] >= 100.0
+    assert stats["clicks_by_provider"].get("openai", 0) >= 1
+    assert stats["conversions_by_provider"].get("openai", 0) >= 1
+    assert stats["revenue_by_provider_usd"].get("openai", 0.0) >= 100.0
+    assert 0 < stats["conversion_rate"] <= 1.0
+    assert stats["period_days"] == 1
+    assert stats["catalog_size"] >= 50
+
+
+def test_referral_stats_empty() -> None:
+    """Stats with no data returns zeroes."""
+    rt = ReferralTracker()
+    stats = rt.get_stats(days_back=1)
+    assert stats["total_clicks"] == 0
+    assert stats["total_conversions"] == 0
+    assert stats["total_revenue_usd"] == 0.0
+    assert stats["conversion_rate"] == 0.0
+
+
+def test_referral_in_app_usage() -> None:
+    """Track in-app usage revenue and verify in database."""
+    from db import ReferralInAppUsage, session_scope
+
+    rt = ReferralTracker()
+    rt.track_in_app_usage("openai", 15.0)
+    with session_scope() as s:
+        usage = s.query(ReferralInAppUsage).filter_by(provider="openai").first()
+        assert usage is not None
+        assert usage.revenue_usd == 15.0
+
+
+def test_referral_catalog() -> None:
+    """Verify get_catalog returns correct data."""
+    rt = ReferralTracker()
+    full = rt.get_catalog()
+    assert len(full) > 0
+    for entry in full:
+        assert "category" in entry
+        assert "providers" in entry
+    llm = rt.get_catalog("llm")
+    assert len(llm) > 0
+    assert llm[0]["name"] == "OpenAI"
+    empty = rt.get_catalog("nonexistent")
+    assert empty == []
+
+
+# ── x402 tests (unchanged) ────────────────────────────────────────
 
 
 def test_x402_payment_request() -> None:
@@ -71,7 +163,7 @@ def test_x402_payment_request() -> None:
     assert body["x402Version"] == 1
     assert "accepts" in body
     base_accept = next(a for a in body["accepts"] if a["network"] == "base")
-    assert base_accept["maxAmountRequired"] == "1000"  # $0.001 = 1000 USDC units
+    assert base_accept["maxAmountRequired"] == "1000"
     assert base_accept["asset"] == "USDC"
     assert "PAYMENT-REQUIRED" in headers
 

tests/test_structure_monitor.py

@@ -7,6 +7,8 @@
 
 from unittest.mock import AsyncMock, patch
 
+import pytest
+
 from structure_monitor import (
     check_selectors,
     get_structure_history,
@@ -14,26 +16,125 @@ from structure_monitor import (
 )
 
 
-def test_check_selectors_empty() -> None:
-    import asyncio
+@pytest.fixture
+def temp_db(monkeypatch, tmp_path):
+    import db as db_module
 
-    result = asyncio.run(check_selectors("https://example.com", []))
+    db_path = tmp_path / "test.db"
+    monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{db_path}")
+    monkeypatch.setenv("PRY_DATA_DIR", str(tmp_path))
+    db_module._engine = None
+    db_module._SessionLocal = None
+    db_module.get_engine()
+    yield
+    db_module._engine = None
+    db_module._SessionLocal = None
+
+
+@pytest.mark.asyncio
+async def test_check_selectors_empty():
+    result = await check_selectors("https://example.com", [])
     assert "selectors" in result
+    assert result["all_matched"] is True
 
 
-def test_get_structure_history_no_history() -> None:
-    result = get_structure_history("https://nonexistent-page-test-123.com")
+@pytest.mark.asyncio
+async def test_get_structure_history_no_history(temp_db):
+    result = await get_structure_history("https://nonexistent-page-test-123.com")
     assert result["has_history"] is False
 
 
-def test_monitor_no_selectors() -> None:
-    import asyncio
-
+@pytest.mark.asyncio
+async def test_monitor_page_structure_creates_history(temp_db):
     mock_client = AsyncMock()
     mock_client.get.return_value.is_success = True
-    mock_client.get.return_value.text = "<html><body>test</body></html>"
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
 
     with patch("structure_monitor.get_client", return_value=mock_client):
-        result = asyncio.run(monitor_page_structure("https://example.com", []))
+        result = await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
     assert "selectors" in result
     assert "changes" in result
+    assert result["all_matched"] is True
+    assert result["has_changes"] is False
+
+
+@pytest.mark.asyncio
+async def test_get_structure_history_with_data(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = True
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
+    result = await get_structure_history("https://example.com")
+    assert result["has_history"] is True
+    assert "last_checked" in result
+    assert "last_selectors" in result
+
+
+@pytest.mark.asyncio
+async def test_monitor_page_structure_with_template_id(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = True
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        result = await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+            template_id="template-1",
+        )
+
+    assert result["template_id"] == "template-1"
+
+    by_url = await get_structure_history("https://example.com")
+    assert by_url["has_history"] is True
+
+    by_template = await get_structure_history("https://example.com", template_id="template-1")
+    assert by_template["has_history"] is True
+
+
+@pytest.mark.asyncio
+async def test_monitor_page_structure_detects_change(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = True
+    mock_client.get.return_value.text = "<html><body><h1>Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
+    mock_client.get.return_value.text = "<html><body><h1>New Title</h1></body></html>"
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        result = await monitor_page_structure(
+            "https://example.com",
+            [{"name": "title", "selector": "h2", "type": "css"}],
+        )
+
+    assert "changes" in result
+
+
+@pytest.mark.asyncio
+async def test_structure_page_http_error(temp_db):
+    mock_client = AsyncMock()
+    mock_client.get.return_value.is_success = False
+    mock_client.get.return_value.status_code = 404
+
+    with patch("structure_monitor.get_client", return_value=mock_client):
+        result = await monitor_page_structure(
+            "https://example.com/404",
+            [{"name": "title", "selector": "h1", "type": "css"}],
+        )
+
+    assert "error" in result