feat(db): add Alembic migrations for versioned schema changes
- Add alembic to dev deps and requirements.txt. - Initialize alembic/ config with SQLite + Postgres support. - Create 0001_initial_schema migration covering all 26 models. - Add db.run_migrations() helper and CLI command. - Add docker-entrypoint.sh to run migrations before app start. - Keep Base.metadata.create_all() for backward compatibility. - Add tests verifying alembic current and upgrade head work.
This commit is contained in:
parent
a3cad41da2
commit
0fc69d2650
12 changed files with 647 additions and 0 deletions
15
=1.14.0
Normal file
15
=1.14.0
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
Collecting alembic
|
||||
Using cached alembic-1.18.5-py3-none-any.whl.metadata (7.2 kB)
|
||||
Requirement already satisfied: SQLAlchemy>=1.4.23 in /tmp/test-pry/lib/python3.12/site-packages (from alembic) (2.0.51)
|
||||
Collecting Mako (from alembic)
|
||||
Using cached mako-1.3.12-py3-none-any.whl.metadata (2.9 kB)
|
||||
Requirement already satisfied: typing-extensions>=4.12 in /tmp/test-pry/lib/python3.12/site-packages (from alembic) (4.16.0)
|
||||
Requirement already satisfied: greenlet>=1 in /tmp/test-pry/lib/python3.12/site-packages (from SQLAlchemy>=1.4.23->alembic) (3.5.3)
|
||||
Collecting MarkupSafe>=0.9.2 (from Mako->alembic)
|
||||
Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB)
|
||||
Using cached alembic-1.18.5-py3-none-any.whl (264 kB)
|
||||
Using cached mako-1.3.12-py3-none-any.whl (78 kB)
|
||||
Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB)
|
||||
Installing collected packages: MarkupSafe, Mako, alembic
|
||||
|
||||
Successfully installed Mako-1.3.12 MarkupSafe-3.0.3 alembic-1.18.5
|
||||
|
|
@ -34,6 +34,9 @@ COPY --from=builder /root/.cache/ms-playwright /root/.cache/ms-playwright
|
|||
|
||||
WORKDIR /app
|
||||
COPY *.py ./
|
||||
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
COPY alembic.ini ./alembic.ini
|
||||
COPY alembic/ ./alembic/
|
||||
COPY routers/ ./routers/
|
||||
COPY llm_providers/ ./llm_providers/
|
||||
COPY stealth_scripts/ ./stealth_scripts/
|
||||
|
|
@ -44,5 +47,6 @@ EXPOSE 8002
|
|||
HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD curl -sf http://localhost:8002/health || exit 1
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8002", \
|
||||
"--workers", "2", "--timeout-keep-alive", "120", "--limit-concurrency", "20"]
|
||||
|
|
|
|||
40
alembic.ini
Normal file
40
alembic.ini
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = sqlite:///tmp/pry-migrations-check.db
|
||||
|
||||
[post_write_hooks]
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
54
alembic/env.py
Normal file
54
alembic/env.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import os
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from alembic import context
|
||||
from db import Base
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
db_url = os.getenv("PRY_DATABASE_URL")
|
||||
if db_url is not None:
|
||||
config.set_main_option("sqlalchemy.url", db_url)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
exclude_tables = {"spatial_ref_sys"}
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
include_exclude_tables=exclude_tables,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
include_exclude_tables=exclude_tables,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
27
alembic/script.py.mako
Normal file
27
alembic/script.py.mako
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
381
alembic/versions/0001_initial_schema.py
Normal file
381
alembic/versions/0001_initial_schema.py
Normal file
|
|
@ -0,0 +1,381 @@
|
|||
"""initial schema — create all 26 model tables
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-07-03 00:00:00.000000
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0001"
|
||||
down_revision: str | None = None
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"quality_checks",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("extraction_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("completeness", sa.Float(), server_default="0.0"),
|
||||
sa.Column("accuracy", sa.Float(), server_default="0.0"),
|
||||
sa.Column("freshness", sa.Float(), server_default="0.0"),
|
||||
sa.Column("overall_score", sa.Float(), server_default="0.0"),
|
||||
sa.Column("risk_level", sa.String(16), server_default="unknown"),
|
||||
sa.Column("issues", sa.JSON(), server_default="[]"),
|
||||
sa.Column("checked_at", sa.DateTime(), index=True),
|
||||
sa.Column("data", sa.JSON(), server_default="{}"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"review_items",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("review_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("fields", sa.JSON(), server_default="{}"),
|
||||
sa.Column("status", sa.String(16), server_default="pending", index=True),
|
||||
sa.Column("reviewer", sa.String(128), server_default=""),
|
||||
sa.Column("notes", sa.Text(), server_default=""),
|
||||
sa.Column("submitted_at", sa.DateTime(), index=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"intel_snapshots",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("competitor_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("competitor_name", sa.String(256), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("fields", sa.JSON(), server_default="{}"),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")])
|
||||
op.create_table(
|
||||
"costing_entries",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("operation", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("cost_usd", sa.Float(), server_default="0.0"),
|
||||
sa.Column("units", sa.Float(), server_default="1.0"),
|
||||
sa.Column("provider", sa.String(64), server_default=""),
|
||||
sa.Column("metadata", sa.JSON(), server_default="{}"),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"freshness_snapshots",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False, index=True),
|
||||
sa.Column("content_hash", sa.String(64), server_default=""),
|
||||
sa.Column("fingerprint", sa.String(128), server_default=""),
|
||||
sa.Column("checked_at", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"structure_snapshots",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False, index=True),
|
||||
sa.Column("url_hash", sa.String(64), server_default="", index=True),
|
||||
sa.Column("template_id", sa.String(128), server_default="", index=True),
|
||||
sa.Column("structure_hash", sa.String(64), server_default=""),
|
||||
sa.Column("selectors", sa.JSON(), server_default="[]"),
|
||||
sa.Column("all_matched", sa.Boolean(), server_default="0"),
|
||||
sa.Column("matched_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("failed_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("changes", sa.JSON(), server_default="[]"),
|
||||
sa.Column("change_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("has_changes", sa.Boolean(), server_default="0"),
|
||||
sa.Column("dom_summary", sa.JSON(), server_default="{}"),
|
||||
sa.Column("checked_at", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"seo_snapshots",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False, index=True),
|
||||
sa.Column("title", sa.Text(), server_default=""),
|
||||
sa.Column("meta_description", sa.Text(), server_default=""),
|
||||
sa.Column("h1", sa.Text(), server_default=""),
|
||||
sa.Column("keyword_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("readability_score", sa.Float(), server_default="0.0"),
|
||||
sa.Column("full_data", sa.JSON(), server_default="{}"),
|
||||
sa.Column("checked_at", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"monitors",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("monitor_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("monitor_name", sa.String(256), server_default=""),
|
||||
sa.Column("schedule_cron", sa.String(64), server_default=""),
|
||||
sa.Column("check_type", sa.String(32), server_default="content"),
|
||||
sa.Column("goal", sa.Text(), server_default=""),
|
||||
sa.Column("use_llm_judge", sa.Boolean(), server_default="0"),
|
||||
sa.Column("active", sa.Boolean(), server_default="1", index=True),
|
||||
sa.Column("webhook_url", sa.Text(), server_default=""),
|
||||
sa.Column("last_check_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_run_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("current_version", sa.Integer(), server_default="0"),
|
||||
sa.Column("last_content", sa.Text(), server_default=""),
|
||||
sa.Column("total_checks", sa.Integer(), server_default="0"),
|
||||
sa.Column("total_changes", sa.Integer(), server_default="0"),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"monitor_runs",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("monitor_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.Column("status", sa.String(16), server_default="success"),
|
||||
sa.Column("diff", sa.JSON(), server_default="{}"),
|
||||
sa.Column("error", sa.Text(), server_default=""),
|
||||
sa.ForeignKeyConstraint(["monitor_id"], ["monitors.monitor_id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"accounts",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("account_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("service", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("username", sa.String(256), nullable=False),
|
||||
sa.Column("email", sa.String(256), server_default=""),
|
||||
sa.Column("status", sa.String(16), server_default="active"),
|
||||
sa.Column("used_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("last_used_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"browser_sessions",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("session_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("cookies", sa.JSON(), server_default="[]"),
|
||||
sa.Column("local_storage", sa.JSON(), server_default="{}"),
|
||||
sa.Column("user_agent", sa.String(512), server_default=""),
|
||||
sa.Column("proxy", sa.String(256), server_default=""),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.Column("expires_at", sa.DateTime(), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"reports",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("report_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("report_name", sa.String(256), server_default=""),
|
||||
sa.Column("report_type", sa.String(64), server_default="html"),
|
||||
sa.Column("url", sa.Text(), server_default=""),
|
||||
sa.Column("data", sa.JSON(), server_default="{}"),
|
||||
sa.Column("generated_at", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"training_datasets",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("dataset_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("dataset_name", sa.String(256), server_default=""),
|
||||
sa.Column("license_class", sa.String(64), server_default="unknown"),
|
||||
sa.Column("pii_redacted", sa.Boolean(), server_default="0"),
|
||||
sa.Column("record_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("file_path", sa.Text(), server_default=""),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"pipelines",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("pipeline_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("pipeline_name", sa.String(256), server_default=""),
|
||||
sa.Column("steps", sa.JSON(), server_default="[]"),
|
||||
sa.Column("schedule_cron", sa.String(64), server_default=""),
|
||||
sa.Column("active", sa.Boolean(), server_default="1"),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"pipeline_runs",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("pipeline_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.Column("status", sa.String(16), server_default="success"),
|
||||
sa.Column("result", sa.JSON(), server_default="{}"),
|
||||
sa.ForeignKeyConstraint(["pipeline_id"], ["pipelines.pipeline_id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"gdpr_requests",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("request_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("request_type", sa.String(32), nullable=False),
|
||||
sa.Column("subject_id", sa.String(256), nullable=False, index=True),
|
||||
sa.Column("status", sa.String(16), server_default="pending"),
|
||||
sa.Column("requested_at", sa.DateTime(), index=True),
|
||||
sa.Column("completed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("details", sa.JSON(), server_default="{}"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"agencies",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("agency_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("agency_name", sa.String(256), nullable=False),
|
||||
sa.Column("owner_email", sa.String(256), nullable=False),
|
||||
sa.Column("custom_domain", sa.String(256), server_default=""),
|
||||
sa.Column("brand_color", sa.String(16), server_default="#f59e0b"),
|
||||
sa.Column("logo_url", sa.Text(), server_default=""),
|
||||
sa.Column("quota", sa.Integer(), server_default="10000"),
|
||||
sa.Column("active", sa.Boolean(), server_default="1"),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"agency_clients",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("client_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("agency_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("email", sa.String(256), server_default=""),
|
||||
sa.Column("quota_used", sa.Integer(), server_default="0"),
|
||||
sa.Column("quota_limit", sa.Integer(), server_default="1000"),
|
||||
sa.Column("active", sa.Boolean(), server_default="1"),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.ForeignKeyConstraint(["agency_id"], ["agencies.agency_id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"referral_clicks",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("click_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("provider", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("source", sa.String(64), server_default=""),
|
||||
sa.Column("user_id", sa.String(128), server_default=""),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.Column("converted", sa.Boolean(), server_default="0"),
|
||||
sa.Column("conversion_value_usd", sa.Float(), server_default="0.0"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"referral_conversions",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("click_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("provider", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("revenue_usd", sa.Float(), server_default="0.0"),
|
||||
sa.Column("notes", sa.Text(), server_default=""),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.ForeignKeyConstraint(["click_id"], ["referral_clicks.click_id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"referral_in_app_usage",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("provider", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("revenue_usd", sa.Float(), server_default="0.0"),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"actors",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("actor_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("name", sa.String(256), nullable=False),
|
||||
sa.Column("description", sa.Text(), server_default=""),
|
||||
sa.Column("template_id", sa.String(64), server_default=""),
|
||||
sa.Column("code", sa.Text(), server_default=""),
|
||||
sa.Column("price_per_run", sa.Float(), server_default="0.0"),
|
||||
sa.Column("visibility", sa.String(16), server_default="private"),
|
||||
sa.Column("schedule_cron", sa.String(64), server_default=""),
|
||||
sa.Column("tags", sa.JSON(), server_default="[]"),
|
||||
sa.Column("author", sa.String(128), server_default=""),
|
||||
sa.Column("run_count", sa.Integer(), server_default="0"),
|
||||
sa.Column("revenue_usd", sa.Float(), server_default="0.0"),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"actor_runs",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("actor_id", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.Column("status", sa.String(16), server_default="success"),
|
||||
sa.Column("result", sa.JSON(), server_default="{}"),
|
||||
sa.Column("cost_usd", sa.Float(), server_default="0.0"),
|
||||
sa.ForeignKeyConstraint(["actor_id"], ["actors.actor_id"]),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"llm_usage",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("provider", sa.String(64), nullable=False, index=True),
|
||||
sa.Column("model", sa.String(128), server_default=""),
|
||||
sa.Column("input_tokens", sa.Integer(), server_default="0"),
|
||||
sa.Column("output_tokens", sa.Integer(), server_default="0"),
|
||||
sa.Column("cost_usd", sa.Float(), server_default="0.0"),
|
||||
sa.Column("operation", sa.String(64), server_default=""),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"webhooks",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("webhook_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("events", sa.JSON(), server_default="[]"),
|
||||
sa.Column("secret", sa.String(128), server_default=""),
|
||||
sa.Column("active", sa.Boolean(), server_default="1"),
|
||||
sa.Column("created_at", sa.DateTime()),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_table(
|
||||
"x402_receipts",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("receipt_id", sa.String(64), nullable=False, unique=True, index=True),
|
||||
sa.Column("tx_hash", sa.String(128), index=True, server_default=""),
|
||||
sa.Column("network", sa.String(32), server_default="base"),
|
||||
sa.Column("asset", sa.String(16), server_default="USDC"),
|
||||
sa.Column("amount_usd", sa.Float(), server_default="0.0"),
|
||||
sa.Column("payer", sa.String(128), server_default=""),
|
||||
sa.Column("payee", sa.String(128), server_default=""),
|
||||
sa.Column("operation", sa.String(64), server_default=""),
|
||||
sa.Column("ts", sa.DateTime(), index=True),
|
||||
sa.Column("facilitator", sa.String(64), server_default="x402.org"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("x402_receipts")
|
||||
op.drop_table("webhooks")
|
||||
op.drop_table("llm_usage")
|
||||
op.drop_table("actor_runs")
|
||||
op.drop_table("actors")
|
||||
op.drop_table("referral_in_app_usage")
|
||||
op.drop_table("referral_conversions")
|
||||
op.drop_table("referral_clicks")
|
||||
op.drop_table("agency_clients")
|
||||
op.drop_table("agencies")
|
||||
op.drop_table("gdpr_requests")
|
||||
op.drop_table("pipeline_runs")
|
||||
op.drop_table("pipelines")
|
||||
op.drop_table("training_datasets")
|
||||
op.drop_table("reports")
|
||||
op.drop_table("browser_sessions")
|
||||
op.drop_table("accounts")
|
||||
op.drop_table("monitor_runs")
|
||||
op.drop_table("monitors")
|
||||
op.drop_table("seo_snapshots")
|
||||
op.drop_table("structure_snapshots")
|
||||
op.drop_table("freshness_snapshots")
|
||||
op.drop_table("costing_entries")
|
||||
op.drop_table("intel_snapshots")
|
||||
op.drop_table("review_items")
|
||||
op.drop_table("quality_checks")
|
||||
11
cli.py
11
cli.py
|
|
@ -193,6 +193,13 @@ def cmd_screenshot(url, output=None, timeout=30):
|
|||
print(f"{GREEN}✓{NC} Screenshot: {len(b64)} bytes (base64)")
|
||||
|
||||
|
||||
def cmd_migrate():
|
||||
"""Run pending database migrations (alembic upgrade head)."""
|
||||
from db import run_migrations
|
||||
run_migrations()
|
||||
print(f"{GREEN}✓{NC} Migrations up to date")
|
||||
|
||||
|
||||
def cmd_run(pryfile_path="pry.yml"):
|
||||
"""Execute jobs defined in pry.yml."""
|
||||
if not os.path.exists(pryfile_path):
|
||||
|
|
@ -261,6 +268,7 @@ def main():
|
|||
print(f" {CYAN}pry parse <url>{NC} Parse a document")
|
||||
print(f" {CYAN}pry ss <url>{NC} Take a screenshot")
|
||||
print(f" {CYAN}pry run [pry.yml]{NC} Execute job file")
|
||||
print(f" {CYAN}pry migrate{NC} Run database migrations")
|
||||
print(f" {CYAN}pry serve{NC} Start the server")
|
||||
print(f" {CYAN}pry completions{NC} Install autocomplete")
|
||||
print(f" {CYAN}pry proxy ...{NC} Manage proxy providers and signup")
|
||||
|
|
@ -329,6 +337,9 @@ def main():
|
|||
port = int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else 8005
|
||||
cmd_serve(port=port)
|
||||
|
||||
elif cmd in ("migrate", "db"):
|
||||
cmd_migrate()
|
||||
|
||||
elif cmd in ("completions", "autocomplete"):
|
||||
shell = args[0] if args else "bash"
|
||||
cmd_completions(shell)
|
||||
|
|
|
|||
25
db.py
25
db.py
|
|
@ -144,6 +144,7 @@ def get_engine() -> Engine:
|
|||
|
||||
_SessionLocal = sessionmaker(bind=_engine, autoflush=False, expire_on_commit=False)
|
||||
# Auto-create tables on first import (idempotent)
|
||||
# Deprecated: use `alembic upgrade head` instead (see run_migrations())
|
||||
Base.metadata.create_all(_engine)
|
||||
logger.info(
|
||||
"db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url}
|
||||
|
|
@ -780,6 +781,30 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
|
|||
return counts
|
||||
|
||||
|
||||
def run_migrations() -> None:
|
||||
"""Run pending Alembic migrations (``alembic upgrade head``).
|
||||
|
||||
Optional helper for programmatic migration execution. Falls back to
|
||||
the config file at *alembic.ini* (must be on the sys.path or in CWD).
|
||||
Requires the ``alembic`` package (included in ``pry[dev]``).
|
||||
"""
|
||||
try:
|
||||
from alembic.config import Config
|
||||
|
||||
from alembic import command
|
||||
except ImportError:
|
||||
logger.warning("alembic_not_installed", extra={"hint": "pip install alembic"})
|
||||
return
|
||||
try:
|
||||
cfg = Config("alembic.ini")
|
||||
cfg.set_main_option("sqlalchemy.url", _resolve_database_url())
|
||||
command.upgrade(cfg, "head")
|
||||
logger.info("migrations_completed")
|
||||
except Exception:
|
||||
logger.exception("migrations_failed")
|
||||
raise
|
||||
|
||||
|
||||
def db_health() -> dict[str, Any]:
|
||||
"""Return a quick health dict for /health or /status endpoints."""
|
||||
if not _HAS_SA:
|
||||
|
|
|
|||
9
docker-entrypoint.sh
Executable file
9
docker-entrypoint.sh
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
if [ "$PRY_SKIP_MIGRATIONS" != "1" ]; then
|
||||
echo "Running database migrations..."
|
||||
alembic upgrade head
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
|
|
@ -60,6 +60,7 @@ dev = [
|
|||
"ruff>=0.7.0",
|
||||
"mypy>=1.12.0",
|
||||
"pre-commit>=4.0.0",
|
||||
"alembic>=1.14.0",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
|
|
|||
|
|
@ -26,3 +26,4 @@ croniter>=2.0.0
|
|||
structlog>=24.0.0
|
||||
sqlalchemy>=2.0.0
|
||||
aiosqlite>=0.19.0
|
||||
alembic>=1.14.0
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from __future__ import annotations
|
|||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -201,6 +202,84 @@ def test_db_health_returns_dict(temp_data_dir):
|
|||
assert "url" in health
|
||||
|
||||
|
||||
def test_alembic_migration_current_works(temp_data_dir, monkeypatch):
|
||||
"""Verify alembic.current can run against a temp SQLite DB without error."""
|
||||
import importlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import db
|
||||
|
||||
monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{temp_data_dir}/migration-test.db")
|
||||
importlib.reload(db)
|
||||
db._engine = None
|
||||
db._SessionLocal = None
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "--config", "alembic.ini", "current"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**os.environ, "PRY_DATABASE_URL": f"sqlite:///{temp_data_dir}/migration-test.db"},
|
||||
)
|
||||
assert result.returncode == 0, f"alembic current failed (stderr): {result.stderr}"
|
||||
|
||||
|
||||
def test_alembic_migration_upgrade_head_creates_tables(temp_data_dir, monkeypatch):
|
||||
"""Verify alembic upgrade head creates all expected tables."""
|
||||
import importlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from sqlalchemy import inspect
|
||||
|
||||
import db
|
||||
|
||||
monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{temp_data_dir}/migration-test.db")
|
||||
importlib.reload(db)
|
||||
db._engine = None
|
||||
db._SessionLocal = None
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "--config", "alembic.ini", "upgrade", "head"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**os.environ, "PRY_DATABASE_URL": f"sqlite:///{temp_data_dir}/migration-test.db"},
|
||||
)
|
||||
assert result.returncode == 0, f"alembic upgrade failed: {result.stderr}"
|
||||
|
||||
eng = db.get_engine()
|
||||
inspector = inspect(eng)
|
||||
tables = inspector.get_table_names()
|
||||
expected_tables = {
|
||||
"quality_checks",
|
||||
"review_items",
|
||||
"intel_snapshots",
|
||||
"costing_entries",
|
||||
"freshness_snapshots",
|
||||
"structure_snapshots",
|
||||
"seo_snapshots",
|
||||
"monitors",
|
||||
"monitor_runs",
|
||||
"accounts",
|
||||
"browser_sessions",
|
||||
"reports",
|
||||
"training_datasets",
|
||||
"pipelines",
|
||||
"pipeline_runs",
|
||||
"gdpr_requests",
|
||||
"agencies",
|
||||
"agency_clients",
|
||||
"referral_clicks",
|
||||
"actors",
|
||||
"actor_runs",
|
||||
"llm_usage",
|
||||
"webhooks",
|
||||
"x402_receipts",
|
||||
}
|
||||
missing = expected_tables - set(tables)
|
||||
assert not missing, f"Tables missing after migration: {missing}"
|
||||
|
||||
|
||||
def test_models_have_unique_constraints(temp_data_dir):
|
||||
"""Tables that map to JSON stores with _id fields should have unique indexes
|
||||
so that re-importing the same data is idempotent (or at least detectable)."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue