diff --git a/Dockerfile b/Dockerfile index 191ccc9..db650c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,9 @@ COPY --from=builder /root/.cache/ms-playwright /root/.cache/ms-playwright WORKDIR /app COPY *.py ./ +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY alembic.ini ./alembic.ini +COPY alembic/ ./alembic/ COPY routers/ ./routers/ COPY llm_providers/ ./llm_providers/ COPY stealth_scripts/ ./stealth_scripts/ @@ -44,5 +47,6 @@ EXPOSE 8002 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \ CMD curl -sf http://localhost:8002/health || exit 1 +ENTRYPOINT ["docker-entrypoint.sh"] CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8002", \ "--workers", "2", "--timeout-keep-alive", "120", "--limit-concurrency", "20"] diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..2bbafa1 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = sqlite:///tmp/pry-migrations-check.db + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..b277d38 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,54 @@ +import os +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, pool + +from alembic import context +from db import Base + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +db_url = os.getenv("PRY_DATABASE_URL") +if db_url is not None: + config.set_main_option("sqlalchemy.url", db_url) + +target_metadata = Base.metadata +exclude_tables = {"spatial_ref_sys"} + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + include_exclude_tables=exclude_tables, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + include_exclude_tables=exclude_tables, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..b1f8b89 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/0001_initial_schema.py b/alembic/versions/0001_initial_schema.py new file mode 100644 index 0000000..11ef9b9 --- /dev/null +++ b/alembic/versions/0001_initial_schema.py @@ -0,0 +1,381 @@ +"""initial schema — create all 26 model tables + +Revision ID: 0001 +Revises: +Create Date: 2026-07-03 00:00:00.000000 + +""" +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "quality_checks", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("extraction_id", sa.String(64), nullable=False, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("completeness", sa.Float(), server_default="0.0"), + sa.Column("accuracy", sa.Float(), server_default="0.0"), + sa.Column("freshness", sa.Float(), server_default="0.0"), + sa.Column("overall_score", sa.Float(), server_default="0.0"), + sa.Column("risk_level", sa.String(16), server_default="unknown"), + sa.Column("issues", sa.JSON(), server_default="[]"), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.Column("data", sa.JSON(), server_default="{}"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "review_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("review_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("fields", sa.JSON(), server_default="{}"), + sa.Column("status", sa.String(16), server_default="pending", index=True), + sa.Column("reviewer", sa.String(128), server_default=""), + sa.Column("notes", sa.Text(), server_default=""), + sa.Column("submitted_at", sa.DateTime(), index=True), + sa.Column("resolved_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "intel_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("competitor_id", sa.String(64), nullable=False, index=True), + sa.Column("competitor_name", sa.String(256), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("fields", sa.JSON(), server_default="{}"), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")]) + op.create_table( + "costing_entries", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("operation", sa.String(64), nullable=False, index=True), + sa.Column("cost_usd", sa.Float(), server_default="0.0"), + sa.Column("units", sa.Float(), server_default="1.0"), + sa.Column("provider", sa.String(64), server_default=""), + sa.Column("metadata", sa.JSON(), server_default="{}"), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "freshness_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False, index=True), + sa.Column("content_hash", sa.String(64), server_default=""), + sa.Column("fingerprint", sa.String(128), server_default=""), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "structure_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False, index=True), + sa.Column("url_hash", sa.String(64), server_default="", index=True), + sa.Column("template_id", sa.String(128), server_default="", index=True), + sa.Column("structure_hash", sa.String(64), server_default=""), + sa.Column("selectors", sa.JSON(), server_default="[]"), + sa.Column("all_matched", sa.Boolean(), server_default="0"), + sa.Column("matched_count", sa.Integer(), server_default="0"), + sa.Column("failed_count", sa.Integer(), server_default="0"), + sa.Column("changes", sa.JSON(), server_default="[]"), + sa.Column("change_count", sa.Integer(), server_default="0"), + sa.Column("has_changes", sa.Boolean(), server_default="0"), + sa.Column("dom_summary", sa.JSON(), server_default="{}"), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "seo_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False, index=True), + sa.Column("title", sa.Text(), server_default=""), + sa.Column("meta_description", sa.Text(), server_default=""), + sa.Column("h1", sa.Text(), server_default=""), + sa.Column("keyword_count", sa.Integer(), server_default="0"), + sa.Column("readability_score", sa.Float(), server_default="0.0"), + sa.Column("full_data", sa.JSON(), server_default="{}"), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "monitors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("monitor_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("monitor_name", sa.String(256), server_default=""), + sa.Column("schedule_cron", sa.String(64), server_default=""), + sa.Column("check_type", sa.String(32), server_default="content"), + sa.Column("goal", sa.Text(), server_default=""), + sa.Column("use_llm_judge", sa.Boolean(), server_default="0"), + sa.Column("active", sa.Boolean(), server_default="1", index=True), + sa.Column("webhook_url", sa.Text(), server_default=""), + sa.Column("last_check_at", sa.DateTime(), nullable=True), + sa.Column("last_run_at", sa.DateTime(), nullable=True), + sa.Column("current_version", sa.Integer(), server_default="0"), + sa.Column("last_content", sa.Text(), server_default=""), + sa.Column("total_checks", sa.Integer(), server_default="0"), + sa.Column("total_changes", sa.Integer(), server_default="0"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "monitor_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("monitor_id", sa.String(64), nullable=False, index=True), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("status", sa.String(16), server_default="success"), + sa.Column("diff", sa.JSON(), server_default="{}"), + sa.Column("error", sa.Text(), server_default=""), + sa.ForeignKeyConstraint(["monitor_id"], ["monitors.monitor_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "accounts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("account_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("service", sa.String(64), nullable=False, index=True), + sa.Column("username", sa.String(256), nullable=False), + sa.Column("email", sa.String(256), server_default=""), + sa.Column("status", sa.String(16), server_default="active"), + sa.Column("used_count", sa.Integer(), server_default="0"), + sa.Column("last_used_at", sa.DateTime(), nullable=True), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "browser_sessions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("session_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("cookies", sa.JSON(), server_default="[]"), + sa.Column("local_storage", sa.JSON(), server_default="{}"), + sa.Column("user_agent", sa.String(512), server_default=""), + sa.Column("proxy", sa.String(256), server_default=""), + sa.Column("created_at", sa.DateTime()), + sa.Column("expires_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "reports", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("report_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("report_name", sa.String(256), server_default=""), + sa.Column("report_type", sa.String(64), server_default="html"), + sa.Column("url", sa.Text(), server_default=""), + sa.Column("data", sa.JSON(), server_default="{}"), + sa.Column("generated_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "training_datasets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("dataset_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("dataset_name", sa.String(256), server_default=""), + sa.Column("license_class", sa.String(64), server_default="unknown"), + sa.Column("pii_redacted", sa.Boolean(), server_default="0"), + sa.Column("record_count", sa.Integer(), server_default="0"), + sa.Column("file_path", sa.Text(), server_default=""), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "pipelines", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("pipeline_name", sa.String(256), server_default=""), + sa.Column("steps", sa.JSON(), server_default="[]"), + sa.Column("schedule_cron", sa.String(64), server_default=""), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "pipeline_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.String(64), nullable=False, index=True), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("status", sa.String(16), server_default="success"), + sa.Column("result", sa.JSON(), server_default="{}"), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipelines.pipeline_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "gdpr_requests", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("request_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("request_type", sa.String(32), nullable=False), + sa.Column("subject_id", sa.String(256), nullable=False, index=True), + sa.Column("status", sa.String(16), server_default="pending"), + sa.Column("requested_at", sa.DateTime(), index=True), + sa.Column("completed_at", sa.DateTime(), nullable=True), + sa.Column("details", sa.JSON(), server_default="{}"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "agencies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("agency_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("agency_name", sa.String(256), nullable=False), + sa.Column("owner_email", sa.String(256), nullable=False), + sa.Column("custom_domain", sa.String(256), server_default=""), + sa.Column("brand_color", sa.String(16), server_default="#f59e0b"), + sa.Column("logo_url", sa.Text(), server_default=""), + sa.Column("quota", sa.Integer(), server_default="10000"), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "agency_clients", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("client_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("agency_id", sa.String(64), nullable=False, index=True), + sa.Column("name", sa.String(256), nullable=False), + sa.Column("email", sa.String(256), server_default=""), + sa.Column("quota_used", sa.Integer(), server_default="0"), + sa.Column("quota_limit", sa.Integer(), server_default="1000"), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.ForeignKeyConstraint(["agency_id"], ["agencies.agency_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "referral_clicks", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("click_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("source", sa.String(64), server_default=""), + sa.Column("user_id", sa.String(128), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("converted", sa.Boolean(), server_default="0"), + sa.Column("conversion_value_usd", sa.Float(), server_default="0.0"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "referral_conversions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("click_id", sa.String(64), nullable=False, index=True), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("revenue_usd", sa.Float(), server_default="0.0"), + sa.Column("notes", sa.Text(), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.ForeignKeyConstraint(["click_id"], ["referral_clicks.click_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "referral_in_app_usage", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("revenue_usd", sa.Float(), server_default="0.0"), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "actors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("actor_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("name", sa.String(256), nullable=False), + sa.Column("description", sa.Text(), server_default=""), + sa.Column("template_id", sa.String(64), server_default=""), + sa.Column("code", sa.Text(), server_default=""), + sa.Column("price_per_run", sa.Float(), server_default="0.0"), + sa.Column("visibility", sa.String(16), server_default="private"), + sa.Column("schedule_cron", sa.String(64), server_default=""), + sa.Column("tags", sa.JSON(), server_default="[]"), + sa.Column("author", sa.String(128), server_default=""), + sa.Column("run_count", sa.Integer(), server_default="0"), + sa.Column("revenue_usd", sa.Float(), server_default="0.0"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "actor_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("actor_id", sa.String(64), nullable=False, index=True), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("status", sa.String(16), server_default="success"), + sa.Column("result", sa.JSON(), server_default="{}"), + sa.Column("cost_usd", sa.Float(), server_default="0.0"), + sa.ForeignKeyConstraint(["actor_id"], ["actors.actor_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "llm_usage", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("model", sa.String(128), server_default=""), + sa.Column("input_tokens", sa.Integer(), server_default="0"), + sa.Column("output_tokens", sa.Integer(), server_default="0"), + sa.Column("cost_usd", sa.Float(), server_default="0.0"), + sa.Column("operation", sa.String(64), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "webhooks", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("webhook_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("events", sa.JSON(), server_default="[]"), + sa.Column("secret", sa.String(128), server_default=""), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "x402_receipts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("receipt_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("tx_hash", sa.String(128), index=True, server_default=""), + sa.Column("network", sa.String(32), server_default="base"), + sa.Column("asset", sa.String(16), server_default="USDC"), + sa.Column("amount_usd", sa.Float(), server_default="0.0"), + sa.Column("payer", sa.String(128), server_default=""), + sa.Column("payee", sa.String(128), server_default=""), + sa.Column("operation", sa.String(64), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("facilitator", sa.String(64), server_default="x402.org"), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("x402_receipts") + op.drop_table("webhooks") + op.drop_table("llm_usage") + op.drop_table("actor_runs") + op.drop_table("actors") + op.drop_table("referral_in_app_usage") + op.drop_table("referral_conversions") + op.drop_table("referral_clicks") + op.drop_table("agency_clients") + op.drop_table("agencies") + op.drop_table("gdpr_requests") + op.drop_table("pipeline_runs") + op.drop_table("pipelines") + op.drop_table("training_datasets") + op.drop_table("reports") + op.drop_table("browser_sessions") + op.drop_table("accounts") + op.drop_table("monitor_runs") + op.drop_table("monitors") + op.drop_table("seo_snapshots") + op.drop_table("structure_snapshots") + op.drop_table("freshness_snapshots") + op.drop_table("costing_entries") + op.drop_table("intel_snapshots") + op.drop_table("review_items") + op.drop_table("quality_checks") diff --git a/cli.py b/cli.py index b544bcb..c51e59b 100755 --- a/cli.py +++ b/cli.py @@ -193,6 +193,13 @@ def cmd_screenshot(url, output=None, timeout=30): print(f"{GREEN}✓{NC} Screenshot: {len(b64)} bytes (base64)") +def cmd_migrate(): + """Run pending database migrations (alembic upgrade head).""" + from db import run_migrations + run_migrations() + print(f"{GREEN}✓{NC} Migrations up to date") + + def cmd_run(pryfile_path="pry.yml"): """Execute jobs defined in pry.yml.""" if not os.path.exists(pryfile_path): @@ -261,6 +268,7 @@ def main(): print(f" {CYAN}pry parse {NC} Parse a document") print(f" {CYAN}pry ss {NC} Take a screenshot") print(f" {CYAN}pry run [pry.yml]{NC} Execute job file") + print(f" {CYAN}pry migrate{NC} Run database migrations") print(f" {CYAN}pry serve{NC} Start the server") print(f" {CYAN}pry completions{NC} Install autocomplete") print(f" {CYAN}pry proxy ...{NC} Manage proxy providers and signup") @@ -329,6 +337,9 @@ def main(): port = int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else 8005 cmd_serve(port=port) + elif cmd in ("migrate", "db"): + cmd_migrate() + elif cmd in ("completions", "autocomplete"): shell = args[0] if args else "bash" cmd_completions(shell) diff --git a/db.py b/db.py index 2f8325c..0b47ae7 100644 --- a/db.py +++ b/db.py @@ -144,6 +144,7 @@ def get_engine() -> Engine: _SessionLocal = sessionmaker(bind=_engine, autoflush=False, expire_on_commit=False) # Auto-create tables on first import (idempotent) + # Deprecated: use `alembic upgrade head` instead (see run_migrations()) Base.metadata.create_all(_engine) logger.info( "db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url} @@ -681,6 +682,30 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: return counts +def run_migrations() -> None: + """Run pending Alembic migrations (``alembic upgrade head``). + + Optional helper for programmatic migration execution. Falls back to + the config file at *alembic.ini* (must be on the sys.path or in CWD). + Requires the ``alembic`` package (included in ``pry[dev]``). + """ + try: + from alembic.config import Config + + from alembic import command + except ImportError: + logger.warning("alembic_not_installed", extra={"hint": "pip install alembic"}) + return + try: + cfg = Config("alembic.ini") + cfg.set_main_option("sqlalchemy.url", _resolve_database_url()) + command.upgrade(cfg, "head") + logger.info("migrations_completed") + except Exception: + logger.exception("migrations_failed") + raise + + def db_health() -> dict[str, Any]: """Return a quick health dict for /health or /status endpoints.""" if not _HAS_SA: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..a85e692 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +if [ "$PRY_SKIP_MIGRATIONS" != "1" ]; then + echo "Running database migrations..." + alembic upgrade head +fi + +exec "$@" diff --git a/pyproject.toml b/pyproject.toml index 0559806..ccc695c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ dev = [ "ruff>=0.7.0", "mypy>=1.12.0", "pre-commit>=4.0.0", + "alembic>=1.14.0", ] [project.scripts] diff --git a/requirements.txt b/requirements.txt index 29647c3..966ffe3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ croniter>=2.0.0 structlog>=24.0.0 sqlalchemy>=2.0.0 aiosqlite>=0.19.0 +alembic>=1.14.0 diff --git a/tests/test_db.py b/tests/test_db.py index f33572e..c299145 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -14,6 +14,7 @@ from __future__ import annotations import importlib import json +import os import pytest @@ -201,6 +202,84 @@ def test_db_health_returns_dict(temp_data_dir): assert "url" in health +def test_alembic_migration_current_works(temp_data_dir, monkeypatch): + """Verify alembic.current can run against a temp SQLite DB without error.""" + import importlib + import subprocess + import sys + + import db + + monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{temp_data_dir}/migration-test.db") + importlib.reload(db) + db._engine = None + db._SessionLocal = None + + result = subprocess.run( + [sys.executable, "-m", "alembic", "--config", "alembic.ini", "current"], + capture_output=True, + text=True, + env={**os.environ, "PRY_DATABASE_URL": f"sqlite:///{temp_data_dir}/migration-test.db"}, + ) + assert result.returncode == 0, f"alembic current failed (stderr): {result.stderr}" + + +def test_alembic_migration_upgrade_head_creates_tables(temp_data_dir, monkeypatch): + """Verify alembic upgrade head creates all expected tables.""" + import importlib + import subprocess + import sys + + from sqlalchemy import inspect + + import db + + monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{temp_data_dir}/migration-test.db") + importlib.reload(db) + db._engine = None + db._SessionLocal = None + + result = subprocess.run( + [sys.executable, "-m", "alembic", "--config", "alembic.ini", "upgrade", "head"], + capture_output=True, + text=True, + env={**os.environ, "PRY_DATABASE_URL": f"sqlite:///{temp_data_dir}/migration-test.db"}, + ) + assert result.returncode == 0, f"alembic upgrade failed: {result.stderr}" + + eng = db.get_engine() + inspector = inspect(eng) + tables = inspector.get_table_names() + expected_tables = { + "quality_checks", + "review_items", + "intel_snapshots", + "costing_entries", + "freshness_snapshots", + "structure_snapshots", + "seo_snapshots", + "monitors", + "monitor_runs", + "accounts", + "browser_sessions", + "reports", + "training_datasets", + "pipelines", + "pipeline_runs", + "gdpr_requests", + "agencies", + "agency_clients", + "referral_clicks", + "actors", + "actor_runs", + "llm_usage", + "webhooks", + "x402_receipts", + } + missing = expected_tables - set(tables) + assert not missing, f"Tables missing after migration: {missing}" + + def test_models_have_unique_constraints(temp_data_dir): """Tables that map to JSON stores with _id fields should have unique indexes so that re-importing the same data is idempotent (or at least detectable)."""