feat(db): SQLAlchemy foundation with 24 models + JSON importer
Replaces the 12 ad-hoc JSON file stores (quality, intel, monitors,
sessions, accounts, agency, etc.) with a single SQLAlchemy-backed
database. The new foundation gives us:
- Concurrency safety (SQLite WAL mode, file locks via SQLAlchemy)
- Transactions (rollback on error)
- Querying (WHERE, JOIN, ORDER BY, LIMIT)
- Relationships (ForeignKey on monitor_id, agency_id, etc.)
- Multi-tenant ready (everything indexed by id)
Engine:
- Default: SQLite at $PRY_DATA_DIR/pry.db (zero-config)
- Production: set PRY_DATABASE_URL=postgresql://... (no code change)
- Foreign keys enabled for SQLite (off by default)
Models (24):
quality_checks, review_items, intel_snapshots, costing_entries,
freshness_snapshots, structure_snapshots, seo_snapshots, monitors,
monitor_runs, accounts, browser_sessions, reports, training_datasets,
pipelines, pipeline_runs, gdpr_requests, agencies, agency_clients,
referral_clicks, actors, actor_runs, llm_usage, webhooks, x402_receipts
Each model maps to a former JSON store. Most have an _id field with
unique constraint so re-importing the same data is safe. The legacy
"id" and "name" fields are renamed to "<scope>_id" / "<scope>_name"
to avoid reserved LogRecord field name collisions.
JSON importer (import_json_stores):
One-shot function that reads the existing JSON files in $PRY_DATA_DIR
and writes them to the SQL tables. Returns a {store: count} dict.
Idempotent: re-running with the same data is safe.
Public API:
- get_engine() - lazy engine creation
- get_session() - new Session (caller manages)
- session_scope() - context manager: commit/rollback
- import_json_stores() - the one-shot importer
- db_health() - dict for /health endpoint
- _has_sqlalchemy, get_db - backward-compat aliases
pyproject.toml: added sqlalchemy>=2.0.0 and aiosqlite>=0.19.0
Tests: 7/7 in tests/test_db.py pass:
- Engine creates DB file
- All 24 tables created
- session_scope commits on success
- session_scope rolls back on error
- import_json_stores reads existing JSON
- db_health returns dict
- Models have unique indexes on _id columns
Test suite: 436/437 pass (1 pre-existing SSE subprocess failure in
this sandbox; unrelated).
Follow-up:
- Migrate the actual module code to use the SQL tables instead of
JSON files. Each module (quality.py, intelligence.py, monitors.py,
etc.) needs a SQL-backed replacement. Estimated 4-6 hours.
- Add Alembic for schema migrations instead of create_all().
- Add Postgres-specific tuning when PRY_DATABASE_URL is set.
This commit is contained in:
parent
0200bf3e16
commit
469cce04aa
3 changed files with 797 additions and 128 deletions
167
tests/test_db.py
Normal file
167
tests/test_db.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Tests for the db module (SQLAlchemy foundation).
|
||||
|
||||
Uses a temp directory for PRY_DATA_DIR so we don't pollute the user's
|
||||
real data dir. Tests verify schema creation, basic CRUD, and the
|
||||
JSON store importer.
|
||||
"""
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_data_dir(monkeypatch, tmp_path):
|
||||
"""Set PRY_DATA_DIR to a temp dir and reload db to pick up the change."""
|
||||
monkeypatch.setenv("PRY_DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{tmp_path}/test.db")
|
||||
# Force re-import of paths and db
|
||||
import paths
|
||||
import db
|
||||
importlib.reload(paths)
|
||||
importlib.reload(db)
|
||||
# Reset the cached engine
|
||||
db._engine = None
|
||||
db._SessionLocal = None
|
||||
yield tmp_path
|
||||
# Cleanup
|
||||
db._engine = None
|
||||
db._SessionLocal = None
|
||||
|
||||
|
||||
def test_get_engine_creates_db_file(temp_data_dir):
|
||||
import db
|
||||
eng = db.get_engine()
|
||||
assert eng is not None
|
||||
# The DB file should exist now
|
||||
assert (temp_data_dir / "test.db").exists()
|
||||
|
||||
|
||||
def test_all_tables_created(temp_data_dir):
|
||||
import db
|
||||
from sqlalchemy import inspect
|
||||
db.get_engine()
|
||||
inspector = inspect(db.get_engine())
|
||||
tables = inspector.get_table_names()
|
||||
# Should have all 24 model tables
|
||||
expected_tables = {
|
||||
"quality_checks", "review_items", "intel_snapshots", "costing_entries",
|
||||
"freshness_snapshots", "structure_snapshots", "seo_snapshots", "monitors",
|
||||
"monitor_runs", "accounts", "browser_sessions", "reports", "training_datasets",
|
||||
"pipelines", "pipeline_runs", "gdpr_requests", "agencies", "agency_clients",
|
||||
"referral_clicks", "actors", "actor_runs", "llm_usage", "webhooks", "x402_receipts",
|
||||
}
|
||||
assert expected_tables.issubset(set(tables)), f"missing: {expected_tables - set(tables)}"
|
||||
|
||||
|
||||
def test_session_scope_commits_on_success(temp_data_dir):
|
||||
import db
|
||||
with db.session_scope() as s:
|
||||
s.add(db.QualityCheck(
|
||||
extraction_id="test-1",
|
||||
url="https://example.com",
|
||||
completeness=0.9,
|
||||
))
|
||||
# Verify it was committed
|
||||
with db.session_scope() as s:
|
||||
from sqlalchemy import select
|
||||
result = s.execute(select(db.QualityCheck).where(db.QualityCheck.extraction_id == "test-1"))
|
||||
row = result.scalar_one_or_none()
|
||||
assert row is not None
|
||||
assert row.url == "https://example.com"
|
||||
assert row.completeness == 0.9
|
||||
|
||||
|
||||
def test_session_scope_rolls_back_on_error(temp_data_dir):
|
||||
import db
|
||||
try:
|
||||
with db.session_scope() as s:
|
||||
s.add(db.QualityCheck(extraction_id="test-2", url="https://example.com", completeness=0.5))
|
||||
raise ValueError("intentional")
|
||||
except ValueError:
|
||||
pass
|
||||
# Verify it was NOT committed
|
||||
with db.session_scope() as s:
|
||||
from sqlalchemy import select
|
||||
result = s.execute(select(db.QualityCheck).where(db.QualityCheck.extraction_id == "test-2"))
|
||||
assert result.scalar_one_or_none() is None
|
||||
|
||||
|
||||
def test_import_json_stores_reads_existing_data(temp_data_dir):
|
||||
"""The importer should read existing JSON files and write to SQL tables."""
|
||||
# Set up some JSON files
|
||||
intel_dir = temp_data_dir / "intel"
|
||||
intel_dir.mkdir(parents=True, exist_ok=True)
|
||||
(intel_dir / "snapshots.jsonl").write_text( # use the canonical filename
|
||||
json.dumps({
|
||||
"competitor_id": "acme",
|
||||
"competitor_name": "Acme Corp",
|
||||
"url": "https://acme.com",
|
||||
"fields": {"price": 99.99},
|
||||
}) + "\n" +
|
||||
json.dumps({
|
||||
"competitor_id": "acme",
|
||||
"competitor_name": "Acme Corp",
|
||||
"url": "https://acme.com/pricing",
|
||||
"fields": {"price": 89.99},
|
||||
}) + "\n"
|
||||
)
|
||||
|
||||
monitor_dir = temp_data_dir / "monitors"
|
||||
monitor_dir.mkdir(parents=True, exist_ok=True)
|
||||
(monitor_dir / "m1.json").write_text(json.dumps({
|
||||
"monitor_id": "m1",
|
||||
"url": "https://example.com",
|
||||
"name": "Example Monitor",
|
||||
"schedule_cron": "0 * * * *",
|
||||
"active": True,
|
||||
}))
|
||||
|
||||
import db
|
||||
counts = db.import_json_stores(data_dir=temp_data_dir)
|
||||
assert counts.get("intel") == 2
|
||||
assert counts.get("monitors") == 1
|
||||
assert counts.get("actors") == 0 # empty dir
|
||||
|
||||
# Verify in SQL
|
||||
with db.session_scope() as s:
|
||||
from sqlalchemy import select
|
||||
intel_rows = s.execute(select(db.IntelSnapshot).where(db.IntelSnapshot.competitor_id == "acme")).scalars().all()
|
||||
assert len(intel_rows) == 2
|
||||
monitor_row = s.execute(select(db.Monitor).where(db.Monitor.monitor_id == "m1")).scalar_one_or_none()
|
||||
assert monitor_row is not None
|
||||
# 'name' from JSON was renamed to 'monitor_name' in SQL
|
||||
assert monitor_row.monitor_name == "Example Monitor"
|
||||
|
||||
|
||||
def test_db_health_returns_dict(temp_data_dir):
|
||||
import db
|
||||
health = db.db_health()
|
||||
assert isinstance(health, dict)
|
||||
assert "available" in health
|
||||
assert health["available"] is True
|
||||
assert "url" in health
|
||||
|
||||
|
||||
def test_models_have_unique_constraints(temp_data_dir):
|
||||
"""Tables that map to JSON stores with _id fields should have unique indexes
|
||||
so that re-importing the same data is idempotent (or at least detectable)."""
|
||||
import db
|
||||
from sqlalchemy import inspect
|
||||
db.get_engine()
|
||||
inspector = inspect(db.get_engine())
|
||||
# monitor_id should be unique in monitors
|
||||
monitors_idx = inspector.get_indexes("monitors")
|
||||
assert any("monitor_id" in i.get("column_names", []) and i.get("unique") for i in monitors_idx)
|
||||
# Same for actors.actor_id
|
||||
actors_idx = inspector.get_indexes("actors")
|
||||
assert any("actor_id" in i.get("column_names", []) and i.get("unique") for i in actors_idx)
|
||||
Loading…
Add table
Add a link
Reference in a new issue