diff --git a/Dockerfile b/Dockerfile index 191ccc9..db650c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,9 @@ COPY --from=builder /root/.cache/ms-playwright /root/.cache/ms-playwright WORKDIR /app COPY *.py ./ +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY alembic.ini ./alembic.ini +COPY alembic/ ./alembic/ COPY routers/ ./routers/ COPY llm_providers/ ./llm_providers/ COPY stealth_scripts/ ./stealth_scripts/ @@ -44,5 +47,6 @@ EXPOSE 8002 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \ CMD curl -sf http://localhost:8002/health || exit 1 +ENTRYPOINT ["docker-entrypoint.sh"] CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8002", \ "--workers", "2", "--timeout-keep-alive", "120", "--limit-concurrency", "20"] diff --git a/STATUS.md b/STATUS.md index a289b41..011c39f 100644 --- a/STATUS.md +++ b/STATUS.md @@ -5,14 +5,14 @@ > Where we are RIGHT NOW. Update before every commit. ## Last Updated -2026-07-03 - Dual-licensing relicense complete (MIT core + BSL 1.1 stealth) +2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed ## Current Status -🟡 pre-production - repo scaffolded, re-license landed, deploy out of sync. See AUDIT.md. +🟡 pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending. ## Deployment Status - **Last deploy**: TBD -- **Current version**: TBD +- **Current version**: 3.0.0-phase0 - **Health**: TBD - **Uptime (30d)**: TBD - **Active branch**: `main` @@ -22,12 +22,18 @@ ## Recent Activity - 2026-07-02: Repository created from `fleet-template` +- 2026-07-03: Split scraping endpoints from `api.py` into `routers/scraping.py` + `deps.py` +- 2026-07-03: Split template endpoints into `routers/templates.py` + added `POST /v1/templates/batch` +- 2026-07-03: Added e2e tests for scraping and template routers (23 new/passing) +- 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`) +- 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart +- 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing ## Known Issues / Tech Debt -- Repo and deploy at /srv/pry/ are out of sync. Deploy runs a 789-line api.py; repo has 4,668-line api.py with 190 endpoints (MCP, x402, auth, etc.). MCP and x402 are written but not deployed. See AUDIT.md. +- Deploy at /srv/pry/ is out of sync with repo (needs pull + restart to activate PRY_API_KEY and router changes) - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage. - State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite. -- PRY_API_KEY, PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - no auth, no x402, no payments. +- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active. - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002. - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..2bbafa1 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = sqlite:///tmp/pry-migrations-check.db + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..b277d38 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,54 @@ +import os +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, pool + +from alembic import context +from db import Base + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +db_url = os.getenv("PRY_DATABASE_URL") +if db_url is not None: + config.set_main_option("sqlalchemy.url", db_url) + +target_metadata = Base.metadata +exclude_tables = {"spatial_ref_sys"} + + +def run_migrations_offline() -> None: + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + include_exclude_tables=exclude_tables, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + include_exclude_tables=exclude_tables, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..b1f8b89 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/0001_initial_schema.py b/alembic/versions/0001_initial_schema.py new file mode 100644 index 0000000..0b95a93 --- /dev/null +++ b/alembic/versions/0001_initial_schema.py @@ -0,0 +1,384 @@ +"""initial schema — create all 26 model tables + +Revision ID: 0001 +Revises: +Create Date: 2026-07-03 00:00:00.000000 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0001" +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "quality_checks", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("extraction_id", sa.String(64), nullable=False, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("completeness", sa.Float(), server_default="0.0"), + sa.Column("accuracy", sa.Float(), server_default="0.0"), + sa.Column("freshness", sa.Float(), server_default="0.0"), + sa.Column("overall_score", sa.Float(), server_default="0.0"), + sa.Column("risk_level", sa.String(16), server_default="unknown"), + sa.Column("issues", sa.JSON(), server_default="[]"), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.Column("data", sa.JSON(), server_default="{}"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "review_items", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("review_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("fields", sa.JSON(), server_default="{}"), + sa.Column("status", sa.String(16), server_default="pending", index=True), + sa.Column("reviewer", sa.String(128), server_default=""), + sa.Column("notes", sa.Text(), server_default=""), + sa.Column("submitted_at", sa.DateTime(), index=True), + sa.Column("resolved_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "intel_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("competitor_id", sa.String(64), nullable=False, index=True), + sa.Column("competitor_name", sa.String(256), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("fields", sa.JSON(), server_default="{}"), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")] + ) + op.create_table( + "costing_entries", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("operation", sa.String(64), nullable=False, index=True), + sa.Column("cost_usd", sa.Float(), server_default="0.0"), + sa.Column("units", sa.Float(), server_default="1.0"), + sa.Column("provider", sa.String(64), server_default=""), + sa.Column("metadata", sa.JSON(), server_default="{}"), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "freshness_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False, index=True), + sa.Column("content_hash", sa.String(64), server_default=""), + sa.Column("fingerprint", sa.String(128), server_default=""), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "structure_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False, index=True), + sa.Column("url_hash", sa.String(64), server_default="", index=True), + sa.Column("template_id", sa.String(128), server_default="", index=True), + sa.Column("structure_hash", sa.String(64), server_default=""), + sa.Column("selectors", sa.JSON(), server_default="[]"), + sa.Column("all_matched", sa.Boolean(), server_default="0"), + sa.Column("matched_count", sa.Integer(), server_default="0"), + sa.Column("failed_count", sa.Integer(), server_default="0"), + sa.Column("changes", sa.JSON(), server_default="[]"), + sa.Column("change_count", sa.Integer(), server_default="0"), + sa.Column("has_changes", sa.Boolean(), server_default="0"), + sa.Column("dom_summary", sa.JSON(), server_default="{}"), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "seo_snapshots", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("url", sa.Text(), nullable=False, index=True), + sa.Column("title", sa.Text(), server_default=""), + sa.Column("meta_description", sa.Text(), server_default=""), + sa.Column("h1", sa.Text(), server_default=""), + sa.Column("keyword_count", sa.Integer(), server_default="0"), + sa.Column("readability_score", sa.Float(), server_default="0.0"), + sa.Column("full_data", sa.JSON(), server_default="{}"), + sa.Column("checked_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "monitors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("monitor_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("monitor_name", sa.String(256), server_default=""), + sa.Column("schedule_cron", sa.String(64), server_default=""), + sa.Column("check_type", sa.String(32), server_default="content"), + sa.Column("goal", sa.Text(), server_default=""), + sa.Column("use_llm_judge", sa.Boolean(), server_default="0"), + sa.Column("active", sa.Boolean(), server_default="1", index=True), + sa.Column("webhook_url", sa.Text(), server_default=""), + sa.Column("last_check_at", sa.DateTime(), nullable=True), + sa.Column("last_run_at", sa.DateTime(), nullable=True), + sa.Column("current_version", sa.Integer(), server_default="0"), + sa.Column("last_content", sa.Text(), server_default=""), + sa.Column("total_checks", sa.Integer(), server_default="0"), + sa.Column("total_changes", sa.Integer(), server_default="0"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "monitor_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("monitor_id", sa.String(64), nullable=False, index=True), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("status", sa.String(16), server_default="success"), + sa.Column("diff", sa.JSON(), server_default="{}"), + sa.Column("error", sa.Text(), server_default=""), + sa.ForeignKeyConstraint(["monitor_id"], ["monitors.monitor_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "accounts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("account_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("service", sa.String(64), nullable=False, index=True), + sa.Column("username", sa.String(256), nullable=False), + sa.Column("email", sa.String(256), server_default=""), + sa.Column("status", sa.String(16), server_default="active"), + sa.Column("used_count", sa.Integer(), server_default="0"), + sa.Column("last_used_at", sa.DateTime(), nullable=True), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "browser_sessions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("session_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("cookies", sa.JSON(), server_default="[]"), + sa.Column("local_storage", sa.JSON(), server_default="{}"), + sa.Column("user_agent", sa.String(512), server_default=""), + sa.Column("proxy", sa.String(256), server_default=""), + sa.Column("created_at", sa.DateTime()), + sa.Column("expires_at", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "reports", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("report_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("report_name", sa.String(256), server_default=""), + sa.Column("report_type", sa.String(64), server_default="html"), + sa.Column("url", sa.Text(), server_default=""), + sa.Column("data", sa.JSON(), server_default="{}"), + sa.Column("generated_at", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "training_datasets", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("dataset_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("dataset_name", sa.String(256), server_default=""), + sa.Column("license_class", sa.String(64), server_default="unknown"), + sa.Column("pii_redacted", sa.Boolean(), server_default="0"), + sa.Column("record_count", sa.Integer(), server_default="0"), + sa.Column("file_path", sa.Text(), server_default=""), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "pipelines", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("pipeline_name", sa.String(256), server_default=""), + sa.Column("steps", sa.JSON(), server_default="[]"), + sa.Column("schedule_cron", sa.String(64), server_default=""), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "pipeline_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("pipeline_id", sa.String(64), nullable=False, index=True), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("status", sa.String(16), server_default="success"), + sa.Column("result", sa.JSON(), server_default="{}"), + sa.ForeignKeyConstraint(["pipeline_id"], ["pipelines.pipeline_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "gdpr_requests", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("request_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("request_type", sa.String(32), nullable=False), + sa.Column("subject_id", sa.String(256), nullable=False, index=True), + sa.Column("status", sa.String(16), server_default="pending"), + sa.Column("requested_at", sa.DateTime(), index=True), + sa.Column("completed_at", sa.DateTime(), nullable=True), + sa.Column("details", sa.JSON(), server_default="{}"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "agencies", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("agency_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("agency_name", sa.String(256), nullable=False), + sa.Column("owner_email", sa.String(256), nullable=False), + sa.Column("custom_domain", sa.String(256), server_default=""), + sa.Column("brand_color", sa.String(16), server_default="#f59e0b"), + sa.Column("logo_url", sa.Text(), server_default=""), + sa.Column("quota", sa.Integer(), server_default="10000"), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "agency_clients", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("client_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("agency_id", sa.String(64), nullable=False, index=True), + sa.Column("name", sa.String(256), nullable=False), + sa.Column("email", sa.String(256), server_default=""), + sa.Column("quota_used", sa.Integer(), server_default="0"), + sa.Column("quota_limit", sa.Integer(), server_default="1000"), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.ForeignKeyConstraint(["agency_id"], ["agencies.agency_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "referral_clicks", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("click_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("source", sa.String(64), server_default=""), + sa.Column("user_id", sa.String(128), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("converted", sa.Boolean(), server_default="0"), + sa.Column("conversion_value_usd", sa.Float(), server_default="0.0"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "referral_conversions", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("click_id", sa.String(64), nullable=False, index=True), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("revenue_usd", sa.Float(), server_default="0.0"), + sa.Column("notes", sa.Text(), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.ForeignKeyConstraint(["click_id"], ["referral_clicks.click_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "referral_in_app_usage", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("revenue_usd", sa.Float(), server_default="0.0"), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "actors", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("actor_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("name", sa.String(256), nullable=False), + sa.Column("description", sa.Text(), server_default=""), + sa.Column("template_id", sa.String(64), server_default=""), + sa.Column("code", sa.Text(), server_default=""), + sa.Column("price_per_run", sa.Float(), server_default="0.0"), + sa.Column("visibility", sa.String(16), server_default="private"), + sa.Column("schedule_cron", sa.String(64), server_default=""), + sa.Column("tags", sa.JSON(), server_default="[]"), + sa.Column("author", sa.String(128), server_default=""), + sa.Column("run_count", sa.Integer(), server_default="0"), + sa.Column("revenue_usd", sa.Float(), server_default="0.0"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "actor_runs", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("actor_id", sa.String(64), nullable=False, index=True), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("status", sa.String(16), server_default="success"), + sa.Column("result", sa.JSON(), server_default="{}"), + sa.Column("cost_usd", sa.Float(), server_default="0.0"), + sa.ForeignKeyConstraint(["actor_id"], ["actors.actor_id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "llm_usage", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("provider", sa.String(64), nullable=False, index=True), + sa.Column("model", sa.String(128), server_default=""), + sa.Column("input_tokens", sa.Integer(), server_default="0"), + sa.Column("output_tokens", sa.Integer(), server_default="0"), + sa.Column("cost_usd", sa.Float(), server_default="0.0"), + sa.Column("operation", sa.String(64), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "webhooks", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("webhook_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("events", sa.JSON(), server_default="[]"), + sa.Column("secret", sa.String(128), server_default=""), + sa.Column("active", sa.Boolean(), server_default="1"), + sa.Column("created_at", sa.DateTime()), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "x402_receipts", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("receipt_id", sa.String(64), nullable=False, unique=True, index=True), + sa.Column("tx_hash", sa.String(128), index=True, server_default=""), + sa.Column("network", sa.String(32), server_default="base"), + sa.Column("asset", sa.String(16), server_default="USDC"), + sa.Column("amount_usd", sa.Float(), server_default="0.0"), + sa.Column("payer", sa.String(128), server_default=""), + sa.Column("payee", sa.String(128), server_default=""), + sa.Column("operation", sa.String(64), server_default=""), + sa.Column("ts", sa.DateTime(), index=True), + sa.Column("facilitator", sa.String(64), server_default="x402.org"), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("x402_receipts") + op.drop_table("webhooks") + op.drop_table("llm_usage") + op.drop_table("actor_runs") + op.drop_table("actors") + op.drop_table("referral_in_app_usage") + op.drop_table("referral_conversions") + op.drop_table("referral_clicks") + op.drop_table("agency_clients") + op.drop_table("agencies") + op.drop_table("gdpr_requests") + op.drop_table("pipeline_runs") + op.drop_table("pipelines") + op.drop_table("training_datasets") + op.drop_table("reports") + op.drop_table("browser_sessions") + op.drop_table("accounts") + op.drop_table("monitor_runs") + op.drop_table("monitors") + op.drop_table("seo_snapshots") + op.drop_table("structure_snapshots") + op.drop_table("freshness_snapshots") + op.drop_table("costing_entries") + op.drop_table("intel_snapshots") + op.drop_table("review_items") + op.drop_table("quality_checks") diff --git a/api.py b/api.py index 323cf94..35f61a5 100644 --- a/api.py +++ b/api.py @@ -27,17 +27,14 @@ from urllib.parse import urljoin, urlparse import httpx import pydantic -import redis import uvicorn from fastapi import Body, FastAPI, Request, Response, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse from pydantic import BaseModel -from advanced import PryAdvanced -from automator import PryAutomator -from cache import ResponseCache from client import close_client, get_client +from deps import advanced, automator, cache, extractor, parser, queue, ratelimiter, scraper from errors import ( ExternalServiceError, InvalidRequestError, @@ -47,16 +44,15 @@ from errors import ( ) from extraction import JsonCssExtractionStrategy, extract_with_chunking from extractor import SchemaExtractor -from jobqueue import JobQueue from mconfig import PryConfig from mcp_production import make_fallback_server, register_all from mcp_sse import mcp_post_message, mcp_sse_endpoint -from parser import DocumentParser from pipeline import HOOK_POINTS, get_pipeline, run_pipeline from pryextras import BatchProcessor, TransformEngine, recorder, streams -from ratelimit import RateLimiter from routers.auth import router as auth_router -from scraper import BlockDetector, PryScraper +from routers.health import router as health_router +from routers.scraping import router as scraping_router +from routers.templates import router as templates_router from settings import settings from x402_middleware import X402Middleware @@ -84,6 +80,11 @@ except ImportError: async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Startup: validate deps. Shutdown: cleanup clients.""" logger.info("pry_startup", version="3.0.0") + if not settings.api_key: + logger.warning( + "pry_api_key_unset", + message="PRY_API_KEY is not set; all protected endpoints are publicly accessible", + ) get_pipeline() # Initialize pipeline yield logger.info("pry_shutdown") @@ -220,16 +221,6 @@ def add_error_handlers(app: FastAPI) -> None: add_error_handlers(app) -scraper = PryScraper() -automator = PryAutomator() -parser = DocumentParser() -extractor = SchemaExtractor() -cache = ResponseCache(capacity=1000) -ratelimiter = RateLimiter(default_rpm=120, burst=200) -queue = JobQueue() -advanced = PryAdvanced(cache=cache) - - # Public paths that don't need authentication _AUTH_PUBLIC_PATHS: set[str] = {"/health", "/live", "/ready", "/docs", "/openapi.json"} @@ -362,32 +353,6 @@ class PryHttpMiddleware: app.add_middleware(PryHttpMiddleware) -# ── Models ── -class ScrapeRequest(BaseModel): - url: str - formats: list[str] | None = None - onlyMainContent: bool | None = True - timeout: int | None = 30 - bypassCloudflare: bool | None = True - jsRender: bool | None = False - jsonSchema: dict[str, str] | None = None - - -class CrawlRequest(BaseModel): - url: str - maxPages: int | None = 10 - maxDepth: int | None = 2 - scrapeOptions: dict[str, Any] | None = None - webhook: str | None = None - - -class MapRequest(BaseModel): - url: str - search: str | None = None - ignoreSitemap: bool | None = True - limit: int | None = 50 - - class AutomateStep(BaseModel): action: str selector: str | None = None @@ -409,81 +374,7 @@ class ParseRequest(BaseModel): timeout: int | None = 60 -# ── Health ── -@app.get("/health", tags=["Health"], summary="Full health check with dependency status") -async def health_check() -> JSONResponse: - """Comprehensive health check — probes Ollama, FlareSolverr, Redis.""" - deps = {"ollama": False, "flaresolverr": False, "redis": False} - - async def check_ollama() -> bool: - try: - c = await get_client() - r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) - return r.is_success - except Exception: # noqa: BLE001 - return False - - async def check_flare() -> bool: - try: - c = await get_client() - r = await c.post( - settings.flaresolverr_url, - json={"cmd": "sessions.list"}, - headers={"Content-Type": "application/json"}, - timeout=3, - ) - return r.is_success - except Exception: # noqa: BLE001 - return False - - async def check_redis() -> bool: - try: - import redis.asyncio as aioredis - - r = aioredis.from_url(settings.redis_url) - await r.ping() - await r.aclose() - return True - except redis.RedisError: - return False - - results = await asyncio.gather( - check_ollama(), check_flare(), check_redis(), return_exceptions=False - ) - deps["ollama"] = results[0] - deps["flaresolverr"] = results[1] - deps["redis"] = results[2] - flaresolverr_ok = deps["flaresolverr"] - status_code = 200 if flaresolverr_ok else 503 - return JSONResponse( - status_code=status_code, - content={ - "status": "ok" if flaresolverr_ok else "degraded", - "version": "3.0.0", - "dependencies": deps, - }, - ) - - -@app.get("/live", tags=["Health"], summary="Kubernetes liveness probe") -async def live() -> dict[str, str]: - """Simple liveness — always returns 200 if the process is running.""" - return {"status": "alive"} - - -@app.get("/ready", tags=["Health"], summary="Kubernetes readiness probe", response_model=None) -async def ready() -> JSONResponse | dict[str, str]: - """Readiness — checks critical dependencies.""" - try: - c = await get_client() - r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) - if r.is_success: - return {"status": "ready"} - except Exception: # noqa: BLE001 - pass - return JSONResponse(status_code=503, content={"status": "not_ready"}) - - +# ── Stats ── @app.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats") async def stats() -> dict[str, Any]: return { @@ -608,107 +499,6 @@ async def freshness_dashboard() -> dict[str, Any]: return {"success": True, "data": get_staleness_dashboard()} -# ── Scrape ── -@app.post("/v1/scrape", tags=["Scraping"], summary="Scrape a single URL") -async def scrape(request: ScrapeRequest) -> dict[str, Any]: - """Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.""" - # Check cache - cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender} - cached = cache.get(request.url, cache_opts) - if cached: - cached["_cached"] = True - return cached - - try: - result = await scraper.scrape( - request.url, - { - "timeout": request.timeout, - "bypass_cloudflare": request.bypassCloudflare, - "js_render": request.jsRender, - "formats": request.formats, - }, - ) - if result.get("status") != "ok": - raise ScrapeError(result.get("error", "Scrape failed")) - - response: dict[str, Any] = { - "success": True, - "data": { - "markdown": result.get("content", ""), - "metadata": { - "url": request.url, - "method": result.get("method", "unknown"), - "title": result.get("title", ""), - "description": result.get("description", ""), - }, - }, - } - - # JSON schema extraction if requested - if request.jsonSchema: - extracted = await extractor.extract(result.get("content", ""), request.jsonSchema) - response["data"]["json"] = extracted - - cache.set(request.url, response, cache_opts) - return response - except PryError: - raise - except Exception as e: - raise ExternalServiceError(str(e)) from e - - -@app.post("/v1/detect-block", tags=["Scraping"], summary="Detect if a site is blocking the scraper") -async def detect_block(url: str = Body(...)) -> dict[str, Any]: - """Detect what kind of anti-bot protection a site is using. - - Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence. - Useful for debugging scraping issues. - """ - detector = BlockDetector() - results = [] - - # Test direct - try: - client = await get_client() - resp = await client.get( - url, - timeout=15, - follow_redirects=True, - headers={ - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/125.0.0.0 Safari/537.36" - ) - }, - ) - detection = detector.detect(resp.text, resp.status_code, dict(resp.headers)) - results.append({"method": "direct", "status": resp.status_code, **detection}) - except (httpx.HTTPError, httpx.RequestError) as e: - results.append({"method": "direct", "error": str(e)}) - - # Test FlareSolverr - try: - async with httpx.AsyncClient(timeout=30) as fs_client: - fs_resp = await fs_client.post( - settings.flaresolverr_url, - json={"cmd": "request.get", "url": url, "maxTimeout": 15000}, - ) - if fs_resp.is_success: - fs_data = fs_resp.json() - fs_html = fs_data.get("solution", {}).get("response", "") - fs_status = fs_data.get("solution", {}).get("status", 0) - detection = detector.detect(fs_html, fs_status) - results.append({"method": "flaresolverr", "status": fs_status, **detection}) - else: - results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"}) - except (httpx.HTTPError, httpx.RequestError) as e: - results.append({"method": "flaresolverr", "error": str(e)}) - - return {"success": True, "data": {"url": url, "results": results}} - - @app.post( "/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system" ) @@ -788,74 +578,6 @@ async def capture_network( } -@app.post("/v1/capture/lazy", tags=["Scraping"], summary="Detect and handle lazy-loaded content") -async def detect_lazy_content( - url: str = Body(...), - auto_scroll: bool = Body(True), - max_scrolls: int = Body(5), -) -> dict[str, Any]: - """Detect lazy loading and infinite scroll patterns on a page. - - Optionally generate JS to auto-scroll and load all content. - """ - from lazy_load import ( - detect_lazy_loading, - generate_load_more_script, - generate_scroll_script, - ) - - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - html = result.get("raw_html", "") - if not html: - client = await get_client() - try: - resp = await client.get( - url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} - ) - html = resp.text - except (httpx.HTTPError, httpx.RequestError): - raise ScrapeError("Could not fetch raw HTML") from None - - detection = detect_lazy_loading(html) - scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else "" - load_more_script = generate_load_more_script() if auto_scroll else "" - - return { - "success": True, - "data": { - "url": url, - "detection": detection, - "has_lazy_content": any(detection.values()), - "scroll_script": scroll_script, - "load_more_script": load_more_script, - }, - } - - -# ── Crawl ── -@app.post("/v1/crawl", tags=["Scraping"], summary="Crawl multiple pages from a URL") -async def crawl(request: CrawlRequest) -> dict[str, Any]: - """Crawl multiple pages from a URL. Supports async webhooks.""" - if request.webhook: - job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook) - task = asyncio.create_task(_run_crawl_job(job_id, request)) - task.add_done_callback(_log_crawl_job_failure) - return {"success": True, "data": {"id": job_id, "status": "pending"}} - - pages = await scraper.crawl( - request.url, - { - "max_pages": request.maxPages, - "max_depth": request.maxDepth, - "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, - }, - ) - return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} - - @app.post( "/v1/crawl/adaptive", tags=["Scraping"], @@ -929,28 +651,6 @@ async def adaptive_crawl( } -async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None: - try: - pages = await scraper.crawl( - request.url, - { - "max_pages": request.maxPages, - "max_depth": request.maxDepth, - }, - ) - await queue.complete_job(job_id, {"pages": pages}) - except Exception as e: - logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url}) - await queue.fail_job(job_id, str(e)) - - -def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None: - """Log unhandled exceptions from crawl job tasks.""" - exc = task.exception() - if exc: - logger.error("crawl_task_unhandled_error", extra={"error": str(exc)}) - - async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any]) -> None: """Fire a webhook notification for watch events.""" try: @@ -964,14 +664,6 @@ async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any logger.exception("watch_webhook_failed", extra={"url": url, "webhook": webhook}) -# ── Map ── -@app.post("/v1/map", tags=["Scraping"], summary="Discover URLs on a site") -async def map_pages(request: MapRequest) -> dict[str, Any]: - """Discover URLs on a site.""" - urls = await scraper.map_urls(request.url, {"limit": request.limit}) - return {"success": True, "data": {"links": urls}} - - # ── Parse (Documents) ── @app.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)") async def parse_document(request: ParseRequest) -> dict[str, Any]: @@ -1392,31 +1084,6 @@ async def restore_session(session_id: str = Body(...)) -> dict[str, Any]: } -# ── Advanced Features (Firecrawl doesn't have these) ── - - -@app.post("/v1/batch", tags=["Batch"], summary="Scrape multiple URLs in parallel") -async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]: - """Scrape multiple URLs in parallel. Firecrawl charges extra for batch.""" - if len(urls) > 50: - raise InvalidRequestError("Max 50 URLs per batch") - tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] - results = await asyncio.gather(*tasks, return_exceptions=True) - pages = [] - for i, r in enumerate(results): - if isinstance(r, BaseException): - pages.append({"url": urls[i], "error": str(r)}) - elif isinstance(r, dict): - pages.append( - { - "url": urls[i], - "markdown": r.get("content", ""), - "method": r.get("method", "unknown"), - } - ) - return {"success": True, "data": {"pages": pages, "total": len(pages)}} - - @app.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs") async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]: """Scrape two URLs and compare their content. Shows additions, deletions, changes.""" @@ -2244,7 +1911,6 @@ async def extract_stable(data: dict[str, Any] = Body(...)) -> dict[str, Any]: result = await scraper.scrape(url, {"bypass_cloudflare": True}) if result.get("status") != "ok": raise ScrapeError(result.get("error") or "Extraction failed") - from extractor import SchemaExtractor ex = SchemaExtractor() extracted = await ex.extract(result.get("content", ""), fields, mode="llm") @@ -3140,6 +2806,9 @@ async def list_schemas() -> dict[str, Any]: # (split into routers/auth.py on the api-router-split refactor) app.include_router(auth_router) +app.include_router(health_router) +app.include_router(scraping_router) +app.include_router(templates_router) # ── Review ── @@ -3786,61 +3455,6 @@ async def detect_tech(url: str = Body(...)) -> dict[str, Any]: return {"success": True, "data": result.get("tech_stack", {})} -# ── Scraper Templates ── - - -@app.get("/v1/templates", tags=["Templates"], summary="List all pre-built scraper templates") -async def list_templates_endpoint() -> dict[str, Any]: - """List all available pre-built scraper templates. - - Templates are one-click extractors for popular websites: - Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc. - """ - from template_engine import list_templates - - templates = list_templates() - # Group by category - categories: dict[str, list[dict[str, Any]]] = {} - for t in templates: - cat = t.get("category", "general") - categories.setdefault(cat, []).append(t) - return { - "success": True, - "data": {"templates": templates, "categories": categories, "total": len(templates)}, - } - - -@app.get("/v1/templates/{template_id}", tags=["Templates"], summary="Get a scraper template") -async def get_template_endpoint(template_id: str) -> dict[str, Any]: - """Get a specific scraper template with full schema details.""" - from template_engine import get_template - - template = get_template(template_id) - if not template: - raise NotFoundError(f"Template not found: {template_id}") - return {"success": True, "data": template} - - -@app.post( - "/v1/templates/execute", tags=["Templates"], summary="Execute a scraper template against a URL" -) -async def execute_template_endpoint( - template_id: str = Body(...), - url: str = Body(...), -) -> dict[str, Any]: - """Execute a pre-built scraper template against any URL. - - Example: use "amazon_product" template with an Amazon product URL - to get structured title, price, rating, description, etc. - - Templates auto-detect the page structure using pre-configured CSS selectors. - """ - from template_engine import execute_template - - result = await execute_template(template_id, url) - return result - - # ── AI Agent Integration ── diff --git a/apify_schema.py b/apify_schema.py new file mode 100644 index 0000000..5f72f39 --- /dev/null +++ b/apify_schema.py @@ -0,0 +1,429 @@ +"""Pry — Apify actor input/output schema builder. + +Provides a fluent, typed API for constructing Apify actor input and output +schemas (JSON Schema compatible with Apify's `INPUT_SCHEMA.json` and +`OUTPUT_SCHEMA.json`). + +Example: + builder = ApifySchemaBuilder("Product Scraper") + builder.add_string("url", "Start URL", required=True) + builder.add_integer("maxResults", "Max Results", default=10, min=1, max=100) + builder.add_enum("format", "Output Format", ["json", "csv", "markdown"], default="json") + schema = builder.build_input_schema() +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +from __future__ import annotations + +from typing import Any + + +class FieldSpec: + """Base class for Apify schema field specifications.""" + + def __init__( + self, + name: str, + title: str, + field_type: str, + description: str = "", + default: Any | None = None, + nullable: bool = False, + ) -> None: + self.name = name + self.title = title + self.field_type = field_type + self.description = description + self.default = default + self.nullable = nullable + + def to_dict(self) -> dict[str, Any]: + spec: dict[str, Any] = { + "title": self.title, + "type": ([self.field_type, "null"] if self.nullable else self.field_type), + } + if self.description: + spec["description"] = self.description + if self.default is not None: + spec["default"] = self.default + return spec + + +class StringField(FieldSpec): + """String field with optional pattern and editor hints.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: str | None = None, + nullable: bool = False, + min_length: int | None = None, + max_length: int | None = None, + pattern: str | None = None, + editor: str | None = None, + ) -> None: + super().__init__(name, title, "string", description, default, nullable) + self.min_length = min_length + self.max_length = max_length + self.pattern = pattern + self.editor = editor + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + if self.min_length is not None: + spec["minLength"] = self.min_length + if self.max_length is not None: + spec["maxLength"] = self.max_length + if self.pattern is not None: + spec["pattern"] = self.pattern + if self.editor is not None: + spec["editor"] = self.editor + return spec + + +class IntegerField(FieldSpec): + """Integer field with optional range.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: int | None = None, + nullable: bool = False, + minimum: int | None = None, + maximum: int | None = None, + ) -> None: + super().__init__(name, title, "integer", description, default, nullable) + self.minimum = minimum + self.maximum = maximum + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + if self.minimum is not None: + spec["minimum"] = self.minimum + if self.maximum is not None: + spec["maximum"] = self.maximum + return spec + + +class NumberField(FieldSpec): + """Number field with optional range.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: float | None = None, + nullable: bool = False, + minimum: float | None = None, + maximum: float | None = None, + ) -> None: + super().__init__(name, title, "number", description, default, nullable) + self.minimum = minimum + self.maximum = maximum + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + if self.minimum is not None: + spec["minimum"] = self.minimum + if self.maximum is not None: + spec["maximum"] = self.maximum + return spec + + +class BooleanField(FieldSpec): + """Boolean field with optional default.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: bool | None = None, + nullable: bool = False, + ) -> None: + super().__init__(name, title, "boolean", description, default, nullable) + + +class EnumField(FieldSpec): + """Enum field restricted to a set of allowed values.""" + + def __init__( + self, + name: str, + title: str, + enum: list[str], + description: str = "", + default: str | None = None, + nullable: bool = False, + ) -> None: + super().__init__(name, title, "string", description, default, nullable) + self.enum = enum + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + spec["enum"] = self.enum + return spec + + +class ArrayField(FieldSpec): + """Array field with item schema.""" + + def __init__( + self, + name: str, + title: str, + items: dict[str, Any], + description: str = "", + default: list[Any] | None = None, + nullable: bool = False, + min_items: int | None = None, + max_items: int | None = None, + ) -> None: + super().__init__(name, title, "array", description, default, nullable) + self.items = items + self.min_items = min_items + self.max_items = max_items + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + spec["items"] = self.items + if self.min_items is not None: + spec["minItems"] = self.min_items + if self.max_items is not None: + spec["maxItems"] = self.max_items + return spec + + +class ObjectField(FieldSpec): + """Object field with nested properties.""" + + def __init__( + self, + name: str, + title: str, + properties: dict[str, Any], + description: str = "", + default: dict[str, Any] | None = None, + nullable: bool = False, + required: list[str] | None = None, + ) -> None: + super().__init__(name, title, "object", description, default, nullable) + self.properties = properties + self.required = required or [] + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + spec["properties"] = self.properties + if self.required: + spec["required"] = self.required + return spec + + +class ApifySchemaBuilder: + """Fluent builder for Apify actor input/output schemas.""" + + def __init__( + self, title: str = "Actor Input", description: str = "", schema_version: int = 1 + ) -> None: + self.title = title + self.description = description + self.schema_version = schema_version + self._fields: dict[str, FieldSpec] = {} + self._required: list[str] = [] + self._prefab: str | None = None + + def add_field(self, field: FieldSpec, required: bool = False) -> ApifySchemaBuilder: + """Add a field specification to the schema.""" + self._fields[field.name] = field + if required: + self._required.append(field.name) + return self + + def add_string( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: str | None = None, + nullable: bool = False, + min_length: int | None = None, + max_length: int | None = None, + pattern: str | None = None, + editor: str | None = None, + ) -> ApifySchemaBuilder: + field = StringField( + name, + title, + description=description, + default=default, + nullable=nullable, + min_length=min_length, + max_length=max_length, + pattern=pattern, + editor=editor, + ) + return self.add_field(field, required=required) + + def add_integer( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: int | None = None, + nullable: bool = False, + minimum: int | None = None, + maximum: int | None = None, + ) -> ApifySchemaBuilder: + field = IntegerField( + name, + title, + description=description, + default=default, + nullable=nullable, + minimum=minimum, + maximum=maximum, + ) + return self.add_field(field, required=required) + + def add_number( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: float | None = None, + nullable: bool = False, + minimum: float | None = None, + maximum: float | None = None, + ) -> ApifySchemaBuilder: + field = NumberField( + name, + title, + description=description, + default=default, + nullable=nullable, + minimum=minimum, + maximum=maximum, + ) + return self.add_field(field, required=required) + + def add_boolean( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: bool | None = None, + nullable: bool = False, + ) -> ApifySchemaBuilder: + field = BooleanField( + name, title, description=description, default=default, nullable=nullable + ) + return self.add_field(field, required=required) + + def add_enum( + self, + name: str, + title: str, + enum: list[str], + description: str = "", + required: bool = False, + default: str | None = None, + nullable: bool = False, + ) -> ApifySchemaBuilder: + field = EnumField( + name, title, enum, description=description, default=default, nullable=nullable + ) + return self.add_field(field, required=required) + + def add_array( + self, + name: str, + title: str, + items: dict[str, Any], + description: str = "", + required: bool = False, + default: list[Any] | None = None, + nullable: bool = False, + min_items: int | None = None, + max_items: int | None = None, + ) -> ApifySchemaBuilder: + field = ArrayField( + name, + title, + items, + description=description, + default=default, + nullable=nullable, + min_items=min_items, + max_items=max_items, + ) + return self.add_field(field, required=required) + + def add_object( + self, + name: str, + title: str, + properties: dict[str, Any], + description: str = "", + required: bool = False, + default: dict[str, Any] | None = None, + nullable: bool = False, + nested_required: list[str] | None = None, + ) -> ApifySchemaBuilder: + field = ObjectField( + name, + title, + properties, + description=description, + default=default, + nullable=nullable, + required=nested_required, + ) + return self.add_field(field, required=required) + + def set_prefab(self, prefab: str) -> ApifySchemaBuilder: + """Set an Apify input schema prefab (e.g., 'START_URLS').""" + self._prefab = prefab + return self + + def build_input_schema(self) -> dict[str, Any]: + """Build an Apify INPUT_SCHEMA.json-compatible dict.""" + schema: dict[str, Any] = { + "title": self.title, + "type": "object", + "schemaVersion": self.schema_version, + "properties": {name: field.to_dict() for name, field in self._fields.items()}, + } + if self.description: + schema["description"] = self.description + if self._required: + schema["required"] = self._required + if self._prefab: + schema["prefab"] = self._prefab + return schema + + def build_output_schema(self) -> dict[str, Any]: + """Build an Apify OUTPUT_SCHEMA.json-compatible dict.""" + schema: dict[str, Any] = { + "title": self.title, + "type": "object", + "schemaVersion": self.schema_version, + "properties": {name: field.to_dict() for name, field in self._fields.items()}, + } + if self.description: + schema["description"] = self.description + if self._required: + schema["required"] = self._required + return schema diff --git a/cli.py b/cli.py index b544bcb..2ec522c 100755 --- a/cli.py +++ b/cli.py @@ -193,6 +193,14 @@ def cmd_screenshot(url, output=None, timeout=30): print(f"{GREEN}✓{NC} Screenshot: {len(b64)} bytes (base64)") +def cmd_migrate(): + """Run pending database migrations (alembic upgrade head).""" + from db import run_migrations + + run_migrations() + print(f"{GREEN}✓{NC} Migrations up to date") + + def cmd_run(pryfile_path="pry.yml"): """Execute jobs defined in pry.yml.""" if not os.path.exists(pryfile_path): @@ -261,6 +269,7 @@ def main(): print(f" {CYAN}pry parse {NC} Parse a document") print(f" {CYAN}pry ss {NC} Take a screenshot") print(f" {CYAN}pry run [pry.yml]{NC} Execute job file") + print(f" {CYAN}pry migrate{NC} Run database migrations") print(f" {CYAN}pry serve{NC} Start the server") print(f" {CYAN}pry completions{NC} Install autocomplete") print(f" {CYAN}pry proxy ...{NC} Manage proxy providers and signup") @@ -329,6 +338,9 @@ def main(): port = int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else 8005 cmd_serve(port=port) + elif cmd in ("migrate", "db"): + cmd_migrate() + elif cmd in ("completions", "autocomplete"): shell = args[0] if args else "bash" cmd_completions(shell) diff --git a/db.py b/db.py index 85d2987..fb684be 100644 --- a/db.py +++ b/db.py @@ -144,6 +144,7 @@ def get_engine() -> Engine: _SessionLocal = sessionmaker(bind=_engine, autoflush=False, expire_on_commit=False) # Auto-create tables on first import (idempotent) + # Deprecated: use `alembic upgrade head` instead (see run_migrations()) Base.metadata.create_all(_engine) logger.info( "db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url} @@ -780,6 +781,30 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: return counts +def run_migrations() -> None: + """Run pending Alembic migrations (``alembic upgrade head``). + + Optional helper for programmatic migration execution. Falls back to + the config file at *alembic.ini* (must be on the sys.path or in CWD). + Requires the ``alembic`` package (included in ``pry[dev]``). + """ + try: + from alembic.config import Config + + from alembic import command + except ImportError: + logger.warning("alembic_not_installed", extra={"hint": "pip install alembic"}) + return + try: + cfg = Config("alembic.ini") + cfg.set_main_option("sqlalchemy.url", _resolve_database_url()) + command.upgrade(cfg, "head") + logger.info("migrations_completed") + except Exception: + logger.exception("migrations_failed") + raise + + def db_health() -> dict[str, Any]: """Return a quick health dict for /health or /status endpoints.""" if not _HAS_SA: diff --git a/deps.py b/deps.py new file mode 100644 index 0000000..93cdb00 --- /dev/null +++ b/deps.py @@ -0,0 +1,36 @@ +"""Shared global instances for Pry API. + +This module initializes the singleton instances used by all API routers +and should be imported by both api.py and any router module that needs +direct access to shared state. + +Usage: + from deps import scraper, cache, automator, ... + +Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +from __future__ import annotations + +from advanced import PryAdvanced +from automator import PryAutomator +from cache import ResponseCache +from extractor import SchemaExtractor +from jobqueue import JobQueue +from mconfig import PryConfig +from parser import DocumentParser +from ratelimit import RateLimiter +from scraper import PryScraper + +config = PryConfig() + +scraper = PryScraper() +automator = PryAutomator() +parser = DocumentParser() +extractor = SchemaExtractor() +cache = ResponseCache(capacity=1000) +ratelimiter = RateLimiter(default_rpm=120, burst=200) +queue = JobQueue() +advanced = PryAdvanced(cache=cache) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..a85e692 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +if [ "$PRY_SKIP_MIGRATIONS" != "1" ]; then + echo "Running database migrations..." + alembic upgrade head +fi + +exec "$@" diff --git a/pyproject.toml b/pyproject.toml index 0559806..ccc695c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ dev = [ "ruff>=0.7.0", "mypy>=1.12.0", "pre-commit>=4.0.0", + "alembic>=1.14.0", ] [project.scripts] diff --git a/requirements.txt b/requirements.txt index 29647c3..966ffe3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,3 +26,4 @@ croniter>=2.0.0 structlog>=24.0.0 sqlalchemy>=2.0.0 aiosqlite>=0.19.0 +alembic>=1.14.0 diff --git a/routers/__init__.py b/routers/__init__.py index fae5afc..c8f45ce 100644 --- a/routers/__init__.py +++ b/routers/__init__.py @@ -42,4 +42,8 @@ Licensed under MIT. See LICENSE. # Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. -__all__: list[str] = [] # populated as routers are added +__all__: list[str] = [ + "auth_router", + "health_router", + "templates_router", +] diff --git a/routers/health.py b/routers/health.py new file mode 100644 index 0000000..4b783e2 --- /dev/null +++ b/routers/health.py @@ -0,0 +1,100 @@ +"""Pry — Health router (liveness, readiness, dependency probes). + +Split from api.py. Replaces the inline /health, /live, /ready endpoints. + +Endpoints (3): + GET /health — Full health check with dependency status + GET /live — Kubernetes liveness probe + GET /ready — Kubernetes readiness probe + +Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +from __future__ import annotations + +import asyncio + +import redis +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from client import get_client +from settings import settings + +router = APIRouter(tags=["Health"]) + + +@router.get("/health", summary="Full health check with dependency status") +async def health_check() -> JSONResponse: + """Comprehensive health check — probes Ollama, FlareSolverr, Redis.""" + deps = {"ollama": False, "flaresolverr": False, "redis": False} + + async def check_ollama() -> bool: + try: + c = await get_client() + r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) + return r.is_success + except Exception: # noqa: BLE001 + return False + + async def check_flare() -> bool: + try: + c = await get_client() + r = await c.post( + settings.flaresolverr_url, + json={"cmd": "sessions.list"}, + headers={"Content-Type": "application/json"}, + timeout=3, + ) + return r.is_success + except Exception: # noqa: BLE001 + return False + + async def check_redis() -> bool: + try: + import redis.asyncio as aioredis + + r = aioredis.from_url(settings.redis_url) + await r.ping() + await r.aclose() + return True + except redis.RedisError: + return False + + results = await asyncio.gather( + check_ollama(), check_flare(), check_redis(), return_exceptions=False + ) + deps["ollama"] = results[0] + deps["flaresolverr"] = results[1] + deps["redis"] = results[2] + flaresolverr_ok = deps["flaresolverr"] + status_code = 200 if flaresolverr_ok else 503 + return JSONResponse( + status_code=status_code, + content={ + "status": "ok" if flaresolverr_ok else "degraded", + "version": "3.0.0", + "dependencies": deps, + }, + ) + + +@router.get("/live", summary="Kubernetes liveness probe") +async def live() -> dict[str, str]: + """Simple liveness — always returns 200 if the process is running.""" + return {"status": "alive"} + + +@router.get("/ready", summary="Kubernetes readiness probe", response_model=None) +async def ready() -> JSONResponse | dict[str, str]: + """Readiness — checks critical dependencies.""" + try: + c = await get_client() + r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) + if r.is_success: + return {"status": "ready"} + except Exception: # noqa: BLE001 + pass + return JSONResponse(status_code=503, content={"status": "not_ready"}) diff --git a/routers/scraping.py b/routers/scraping.py new file mode 100644 index 0000000..1d7250d --- /dev/null +++ b/routers/scraping.py @@ -0,0 +1,301 @@ +"""Pry — Scraping router (scrape, crawl, map, batch, detect). + +Split from api.py on the api-router-split refactor. Replaces inline +Scraping-tagged endpoints from api.py. Behavior is identical. + +Endpoints (6): + POST /v1/scrape — Scrape a single URL + POST /v1/detect-block — Detect anti-bot protection + POST /v1/capture/lazy — Detect lazy-loaded content + POST /v1/crawl — Crawl multiple pages + POST /v1/map — Discover URLs on a site + POST /v1/batch — Scrape multiple URLs in parallel + +Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. See LICENSE. +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +import httpx +from fastapi import APIRouter, Body +from pydantic import BaseModel + +from client import get_client +from deps import cache, extractor, queue, scraper +from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError +from scraper import BlockDetector +from settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Scraping"]) + + +# ── Models ── + + +class ScrapeRequest(BaseModel): + url: str + formats: list[str] | None = None + onlyMainContent: bool | None = True + timeout: int | None = 30 + bypassCloudflare: bool | None = True + jsRender: bool | None = False + jsonSchema: dict[str, str] | None = None + + +class CrawlRequest(BaseModel): + url: str + maxPages: int | None = 10 + maxDepth: int | None = 2 + scrapeOptions: dict[str, Any] | None = None + webhook: str | None = None + + +class MapRequest(BaseModel): + url: str + search: str | None = None + ignoreSitemap: bool | None = True + limit: int | None = 50 + + +# ── Internal helpers ── + + +async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None: + try: + pages = await scraper.crawl( + request.url, + { + "max_pages": request.maxPages, + "max_depth": request.maxDepth, + }, + ) + await queue.complete_job(job_id, {"pages": pages}) + except Exception as e: + logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url}) + await queue.fail_job(job_id, str(e)) + + +def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None: + """Log unhandled exceptions from crawl job tasks.""" + exc = task.exception() + if exc: + logger.error("crawl_task_unhandled_error", extra={"error": str(exc)}) + + +# ── Scrape ── + + +@router.post("/v1/scrape", summary="Scrape a single URL") +async def scrape(request: ScrapeRequest) -> dict[str, Any]: + """Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.""" + # Check cache + cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender} + cached = cache.get(request.url, cache_opts) + if cached: + cached["_cached"] = True + return cached + + try: + result = await scraper.scrape( + request.url, + { + "timeout": request.timeout, + "bypass_cloudflare": request.bypassCloudflare, + "js_render": request.jsRender, + "formats": request.formats, + }, + ) + if result.get("status") != "ok": + raise ScrapeError(result.get("error", "Scrape failed")) + + response: dict[str, Any] = { + "success": True, + "data": { + "markdown": result.get("content", ""), + "metadata": { + "url": request.url, + "method": result.get("method", "unknown"), + "title": result.get("title", ""), + "description": result.get("description", ""), + }, + }, + } + + # JSON schema extraction if requested + if request.jsonSchema: + extracted = await extractor.extract(result.get("content", ""), request.jsonSchema) + response["data"]["json"] = extracted + + cache.set(request.url, response, cache_opts) + return response + except PryError: + raise + except Exception as e: + raise ExternalServiceError(str(e)) from e + + +@router.post("/v1/detect-block", summary="Detect if a site is blocking the scraper") +async def detect_block(url: str = Body(...)) -> dict[str, Any]: + """Detect what kind of anti-bot protection a site is using. + + Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence. + Useful for debugging scraping issues. + """ + detector = BlockDetector() + results = [] + + # Test direct + try: + client = await get_client() + resp = await client.get( + url, + timeout=15, + follow_redirects=True, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/125.0.0.0 Safari/537.36" + ) + }, + ) + detection = detector.detect(resp.text, resp.status_code, dict(resp.headers)) + results.append({"method": "direct", "status": resp.status_code, **detection}) + except (httpx.HTTPError, httpx.RequestError) as e: + results.append({"method": "direct", "error": str(e)}) + + # Test FlareSolverr + try: + async with httpx.AsyncClient(timeout=30) as fs_client: + fs_resp = await fs_client.post( + settings.flaresolverr_url, + json={"cmd": "request.get", "url": url, "maxTimeout": 15000}, + ) + if fs_resp.is_success: + fs_data = fs_resp.json() + fs_html = fs_data.get("solution", {}).get("response", "") + fs_status = fs_data.get("solution", {}).get("status", 0) + detection = detector.detect(fs_html, fs_status) + results.append({"method": "flaresolverr", "status": fs_status, **detection}) + else: + results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"}) + except (httpx.HTTPError, httpx.RequestError) as e: + results.append({"method": "flaresolverr", "error": str(e)}) + + return {"success": True, "data": {"url": url, "results": results}} + + +@router.post("/v1/capture/lazy", summary="Detect and handle lazy-loaded content") +async def detect_lazy_content( + url: str = Body(...), + auto_scroll: bool = Body(True), + max_scrolls: int = Body(5), +) -> dict[str, Any]: + """Detect lazy loading and infinite scroll patterns on a page. + + Optionally generate JS to auto-scroll and load all content. + """ + from lazy_load import ( + detect_lazy_loading, + generate_load_more_script, + generate_scroll_script, + ) + + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + html = result.get("raw_html", "") + if not html: + client = await get_client() + try: + resp = await client.get( + url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} + ) + html = resp.text + except (httpx.HTTPError, httpx.RequestError): + raise ScrapeError("Could not fetch raw HTML") from None + + detection = detect_lazy_loading(html) + scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else "" + load_more_script = generate_load_more_script() if auto_scroll else "" + + return { + "success": True, + "data": { + "url": url, + "detection": detection, + "has_lazy_content": any(detection.values()), + "scroll_script": scroll_script, + "load_more_script": load_more_script, + }, + } + + +# ── Crawl ── + + +@router.post("/v1/crawl", summary="Crawl multiple pages from a URL") +async def crawl(request: CrawlRequest) -> dict[str, Any]: + """Crawl multiple pages from a URL. Supports async webhooks.""" + if request.webhook: + job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook) + task = asyncio.create_task(_run_crawl_job(job_id, request)) + task.add_done_callback(_log_crawl_job_failure) + return {"success": True, "data": {"id": job_id, "status": "pending"}} + + pages = await scraper.crawl( + request.url, + { + "max_pages": request.maxPages, + "max_depth": request.maxDepth, + "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, + }, + ) + return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} + + +# ── Map ── + + +@router.post("/v1/map", summary="Discover URLs on a site") +async def map_pages(request: MapRequest) -> dict[str, Any]: + """Discover URLs on a site.""" + urls = await scraper.map_urls(request.url, {"limit": request.limit}) + return {"success": True, "data": {"links": urls}} + + +# ── Batch ── + + +@router.post("/v1/batch", summary="Scrape multiple URLs in parallel") +async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]: + """Scrape multiple URLs in parallel. Firecrawl charges extra for batch.""" + if len(urls) > 50: + raise InvalidRequestError("Max 50 URLs per batch") + tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] + results = await asyncio.gather(*tasks, return_exceptions=True) + pages = [] + for i, r in enumerate(results): + if isinstance(r, BaseException): + pages.append({"url": urls[i], "error": str(r)}) + elif isinstance(r, dict): + pages.append( + { + "url": urls[i], + "markdown": r.get("content", ""), + "method": r.get("method", "unknown"), + } + ) + return {"success": True, "data": {"pages": pages, "total": len(pages)}} diff --git a/routers/templates.py b/routers/templates.py new file mode 100644 index 0000000..3e4fb8e --- /dev/null +++ b/routers/templates.py @@ -0,0 +1,131 @@ +"""Pry — Templates router.""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from fastapi import APIRouter, Body +from pydantic import BaseModel + +from errors import InvalidRequestError, NotFoundError +from template_engine import execute_template, get_template, list_templates + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Templates"]) + + +class BatchItem(BaseModel): + """A single template execution request within a batch.""" + + template_id: str + url: str + options: dict[str, Any] | None = None + + +class BatchResponse(BaseModel): + """Batch execution result.""" + + success: bool = True + data: dict[str, Any] + + +MAX_BATCH_SIZE = 20 + + +@router.get("/v1/templates", summary="List all pre-built scraper templates") +async def list_templates_endpoint() -> dict[str, Any]: + """List all available pre-built scraper templates. + + Templates are one-click extractors for popular websites: + Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc. + """ + templates = list_templates() + categories: dict[str, list[dict[str, Any]]] = {} + for t in templates: + cat = t.get("category", "general") + categories.setdefault(cat, []).append(t) + return { + "success": True, + "data": {"templates": templates, "categories": categories, "total": len(templates)}, + } + + +@router.get("/v1/templates/{template_id}", summary="Get a scraper template") +async def get_template_endpoint(template_id: str) -> dict[str, Any]: + """Get a specific scraper template with full schema details.""" + template = get_template(template_id) + if not template: + raise NotFoundError(f"Template not found: {template_id}") + return {"success": True, "data": template} + + +@router.post("/v1/templates/execute", summary="Execute a scraper template against a URL") +async def execute_template_endpoint( + template_id: str = Body(...), + url: str = Body(...), +) -> dict[str, Any]: + """Execute a pre-built scraper template against any URL. + + Example: use "amazon_product" template with an Amazon product URL + to get structured title, price, rating, description, etc. + + Templates auto-detect the page structure using pre-configured CSS selectors. + """ + result = await execute_template(template_id, url) + return result + + +@router.post("/v1/templates/batch", summary="Execute multiple templates/URLs in parallel") +async def batch_execute(body: list[BatchItem]) -> dict[str, Any]: + """Execute multiple scraper templates against multiple URLs in parallel. + + Runs up to 20 template executions concurrently. Each result preserves the + original template_id and url for correlation. + """ + if len(body) > MAX_BATCH_SIZE: + raise InvalidRequestError(f"Max {MAX_BATCH_SIZE} items per batch, got {len(body)}") + + async def run_item(item: BatchItem) -> dict[str, Any]: + try: + result = await execute_template(item.template_id, item.url, **(item.options or {})) + return { + "template_id": item.template_id, + "url": item.url, + "result": result, + } + except Exception as e: # noqa: BLE001 + return { + "template_id": item.template_id, + "url": item.url, + "error": {"code": "execution_failed", "message": str(e)}, + } + + results = await asyncio.gather(*[run_item(item) for item in body], return_exceptions=True) + + flat: list[dict[str, Any]] = [] + failed = 0 + for r in results: + if isinstance(r, Exception): + flat.append({"error": {"code": "execution_failed", "message": str(r)}}) + failed += 1 + else: + if "error" in r: + failed += 1 + flat.append(r) + + return { + "success": True, + "data": { + "results": flat, + "total": len(flat), + "failed": failed, + }, + } diff --git a/tests/test_api.py b/tests/test_api.py index 8c14825..54cb3b4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,14 +2,26 @@ # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper -# Licensed under MIT. See LICENSE. -"""Tests for API helpers.""" +# Licensed under MIT. +"""Tests for API helpers and auth.""" -from api import _get_or_key +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from api import app + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) def test_get_or_key_from_env(monkeypatch) -> None: monkeypatch.setattr("api.settings.openrouter_api_key", "sk-test-key-123") + from api import _get_or_key + key = _get_or_key() assert key == "sk-test-key-123" @@ -17,6 +29,8 @@ def test_get_or_key_from_env(monkeypatch) -> None: def test_get_or_key_empty_when_not_set(monkeypatch) -> None: monkeypatch.setattr("api.settings.openrouter_api_key", "") monkeypatch.setattr("os.path.isfile", lambda p: False) + from api import _get_or_key + key = _get_or_key() assert key is None @@ -26,3 +40,39 @@ def test_vision_models_fallback_list() -> None: assert len(VISION_MODELS_FALLBACK) >= 3 assert all("free" in m for m in VISION_MODELS_FALLBACK) + + +@pytest.mark.asyncio +async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> None: + """Protected endpoints require a valid Bearer PRY_API_KEY.""" + monkeypatch.setattr("api.settings.api_key", "secret-pry-key") + # Disable x402 payment gating so we can test auth in isolation. + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): + resp = client.post("/v1/scrape", json={"url": "https://example.com"}) + assert resp.status_code == 401 + + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): + resp = client.post( + "/v1/scrape", + json={"url": "https://example.com"}, + headers={"authorization": "Bearer wrong-key"}, + ) + assert resp.status_code == 401 + + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): + resp = client.post( + "/v1/scrape", + json={"url": "https://example.com"}, + headers={"authorization": "Bearer secret-pry-key"}, + ) + # auth passes; endpoint returns 402 because we did not mock scraper, which is fine + assert resp.status_code in (200, 402) diff --git a/tests/test_api_scraping.py b/tests/test_api_scraping.py new file mode 100644 index 0000000..1d15549 --- /dev/null +++ b/tests/test_api_scraping.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +"""End-to-end tests for the scraping router endpoints. + +These tests exercise the full FastAPI request/response lifecycle via +TestClient. External I/O (HTTP requests, browser automation) is mocked at the +`deps.scraper` and `deps.extractor` boundaries so tests stay fast and +deterministic. +""" + +from collections.abc import Iterator +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + +from api import app + + +@pytest.fixture(autouse=True) +def bypass_x402_middleware() -> Iterator[None]: + """Disable x402 payment gating so scraping endpoints can be tested.""" + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): + yield + + +@pytest.fixture(autouse=True) +def clear_response_cache() -> Iterator[None]: + """Clear the shared response cache so tests don't see each other's data.""" + from deps import cache as response_cache + + response_cache.clear() + yield + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) + + +@pytest.fixture +def mock_scrape_result() -> dict: + return { + "status": "ok", + "url": "https://example.com", + "title": "Example", + "description": "An example page", + "content": "Hello world", + "markdown": "# Hello world", + "raw_html": "Hello world", + "links": ["https://example.com/a", "https://example.com/b"], + } + + +@pytest.mark.asyncio +async def test_v1_scrape_success(client: TestClient, mock_scrape_result: dict) -> None: + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + resp = client.post("/v1/scrape", json={"url": "https://example.com"}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["metadata"]["url"] == "https://example.com" + assert data["data"]["markdown"] == "Hello world" + + +@pytest.mark.asyncio +async def test_v1_scrape_with_json_schema(client: TestClient, mock_scrape_result: dict) -> None: + schema = {"title": "page title"} + extracted = {"title": "Example"} + with ( + patch("routers.scraping.scraper") as mock_scraper, + patch("routers.scraping.extractor") as mock_extractor, + ): + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + mock_extractor.extract = AsyncMock(return_value=extracted) + resp = client.post("/v1/scrape", json={"url": "https://example.com", "jsonSchema": schema}) + + assert resp.status_code == 200 + data = resp.json() + assert data["data"]["json"] == extracted + + +@pytest.mark.asyncio +async def test_v1_detect_block_success(client: TestClient) -> None: + html = "ok" + with patch("routers.scraping.get_client") as mock_get_client: + mock_client = AsyncMock() + mock_client.get = AsyncMock( + return_value=AsyncMock( + text=html, + status_code=200, + headers={"content-type": "text/html"}, + ) + ) + mock_get_client.return_value = mock_client + resp = client.post( + "/v1/detect-block", + content='"https://example.com"', + headers={"content-type": "application/json"}, + ) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["url"] == "https://example.com" + assert any(r["method"] == "direct" for r in data["data"]["results"]) + + +@pytest.mark.asyncio +async def test_v1_capture_lazy_success(client: TestClient, mock_scrape_result: dict) -> None: + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + resp = client.post("/v1/capture/lazy", json={"url": "https://example.com"}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert "detection" in data["data"] + assert "has_lazy_content" in data["data"] + + +@pytest.mark.asyncio +async def test_v1_crawl_success(client: TestClient, mock_scrape_result: dict) -> None: + pages = [mock_scrape_result] + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.crawl = AsyncMock(return_value=pages) + resp = client.post( + "/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1} + ) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert "pages" in data["data"] + assert data["data"]["url"] == "https://example.com" + + +@pytest.mark.asyncio +async def test_v1_map_success(client: TestClient) -> None: + urls = ["https://example.com/page1", "https://example.com/page2"] + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.map_urls = AsyncMock(return_value=urls) + resp = client.post("/v1/map", json={"url": "https://example.com", "limit": 10}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["links"] == urls + + +@pytest.mark.asyncio +async def test_v1_batch_success(client: TestClient, mock_scrape_result: dict) -> None: + urls = ["https://example.com/1", "https://example.com/2"] + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + resp = client.post("/v1/batch", json=urls) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert len(data["data"]["pages"]) == len(urls) + + +@pytest.mark.asyncio +async def test_v1_batch_requires_urls(client: TestClient) -> None: + resp = client.post("/v1/batch", json={"not_a_list": True}) + assert resp.status_code == 422 diff --git a/tests/test_api_templates.py b/tests/test_api_templates.py new file mode 100644 index 0000000..6b23efa --- /dev/null +++ b/tests/test_api_templates.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +"""End-to-end tests for the templates router endpoints.""" + +from collections.abc import Iterator +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + +from api import app + + +@pytest.fixture(autouse=True) +def bypass_x402_middleware() -> Iterator[None]: + """Disable x402 payment gating for template execution tests.""" + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): + yield + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) + + +@pytest.mark.asyncio +async def test_list_templates(client: TestClient) -> None: + templates = [ + {"id": "amazon_product", "name": "Amazon Product", "category": "ecommerce"}, + {"id": "github_repo", "name": "GitHub Repo", "category": "dev"}, + ] + with patch("routers.templates.list_templates", return_value=templates): + resp = client.get("/v1/templates") + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["total"] == 2 + assert "ecommerce" in data["data"]["categories"] + + +@pytest.mark.asyncio +async def test_get_template(client: TestClient) -> None: + template = {"id": "amazon_product", "name": "Amazon Product"} + with patch("routers.templates.get_template", return_value=template): + resp = client.get("/v1/templates/amazon_product") + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["id"] == "amazon_product" + + +@pytest.mark.asyncio +async def test_get_template_not_found(client: TestClient) -> None: + with patch("routers.templates.get_template", return_value=None): + resp = client.get("/v1/templates/missing") + + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_execute_template(client: TestClient) -> None: + result = {"success": True, "data": {"title": "Widget"}} + with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)): + resp = client.post( + "/v1/templates/execute", + json={"template_id": "amazon_product", "url": "https://example.com"}, + ) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["title"] == "Widget" + + +@pytest.mark.asyncio +async def test_batch_execute_templates(client: TestClient) -> None: + result = {"success": True, "data": {"title": "Widget"}} + items = [ + {"template_id": "amazon_product", "url": "https://example.com/1"}, + {"template_id": "github_repo", "url": "https://example.com/2"}, + ] + with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)): + resp = client.post("/v1/templates/batch", json=items) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["total"] == 2 + assert data["data"]["failed"] == 0 + assert data["data"]["results"][0]["template_id"] == "amazon_product" + + +@pytest.mark.asyncio +async def test_batch_execute_templates_limits_size(client: TestClient) -> None: + items = [{"template_id": "t", "url": "https://example.com"} for _ in range(21)] + resp = client.post("/v1/templates/batch", json=items) + + assert resp.status_code == 400 diff --git a/tests/test_apify_schema.py b/tests/test_apify_schema.py new file mode 100644 index 0000000..80a0848 --- /dev/null +++ b/tests/test_apify_schema.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +"""Tests for the Apify schema builder.""" + +from apify_schema import ApifySchemaBuilder + + +def test_input_schema_basic() -> None: + schema = ( + ApifySchemaBuilder("Product Scraper", "Scrape product details") + .add_string("url", "Start URL", description="The URL to scrape", required=True) + .add_integer("maxResults", "Max Results", default=10, minimum=1, maximum=100) + .add_enum("format", "Output Format", ["json", "csv", "markdown"], default="json") + .build_input_schema() + ) + + assert schema["title"] == "Product Scraper" + assert schema["description"] == "Scrape product details" + assert schema["type"] == "object" + assert schema["schemaVersion"] == 1 + assert schema["required"] == ["url"] + assert schema["properties"]["url"]["type"] == "string" + assert schema["properties"]["maxResults"]["default"] == 10 + assert schema["properties"]["format"]["enum"] == ["json", "csv", "markdown"] + + +def test_output_schema_includes_required() -> None: + schema = ( + ApifySchemaBuilder("Product Output") + .add_string("title", "Title", required=True) + .add_number("price", "Price", minimum=0) + .add_boolean("inStock", "In Stock", default=True) + .build_output_schema() + ) + + assert schema["properties"]["title"]["type"] == "string" + assert schema["properties"]["price"]["minimum"] == 0 + assert schema["properties"]["inStock"]["default"] is True + assert schema["required"] == ["title"] + + +def test_array_and_object_fields() -> None: + schema = ( + ApifySchemaBuilder("Complex Actor") + .add_array("urls", "URLs", {"type": "string"}, required=True, min_items=1, max_items=50) + .add_object( + "proxy", + "Proxy Config", + {"host": {"type": "string"}, "port": {"type": "integer"}}, + nested_required=["host"], + ) + .build_input_schema() + ) + + assert schema["properties"]["urls"]["type"] == "array" + assert schema["properties"]["urls"]["minItems"] == 1 + assert schema["properties"]["urls"]["items"]["type"] == "string" + assert schema["properties"]["proxy"]["type"] == "object" + assert schema["properties"]["proxy"]["required"] == ["host"] + + +def test_nullable_field() -> None: + schema = ( + ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema() + ) + assert schema["properties"]["optional"]["type"] == ["string", "null"] + + +def test_prefab() -> None: + schema = ( + ApifySchemaBuilder() + .set_prefab("START_URLS") + .add_string("url", "URL", required=True) + .build_input_schema() + ) + assert schema["prefab"] == "START_URLS" diff --git a/tests/test_db.py b/tests/test_db.py index f33572e..c299145 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -14,6 +14,7 @@ from __future__ import annotations import importlib import json +import os import pytest @@ -201,6 +202,84 @@ def test_db_health_returns_dict(temp_data_dir): assert "url" in health +def test_alembic_migration_current_works(temp_data_dir, monkeypatch): + """Verify alembic.current can run against a temp SQLite DB without error.""" + import importlib + import subprocess + import sys + + import db + + monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{temp_data_dir}/migration-test.db") + importlib.reload(db) + db._engine = None + db._SessionLocal = None + + result = subprocess.run( + [sys.executable, "-m", "alembic", "--config", "alembic.ini", "current"], + capture_output=True, + text=True, + env={**os.environ, "PRY_DATABASE_URL": f"sqlite:///{temp_data_dir}/migration-test.db"}, + ) + assert result.returncode == 0, f"alembic current failed (stderr): {result.stderr}" + + +def test_alembic_migration_upgrade_head_creates_tables(temp_data_dir, monkeypatch): + """Verify alembic upgrade head creates all expected tables.""" + import importlib + import subprocess + import sys + + from sqlalchemy import inspect + + import db + + monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{temp_data_dir}/migration-test.db") + importlib.reload(db) + db._engine = None + db._SessionLocal = None + + result = subprocess.run( + [sys.executable, "-m", "alembic", "--config", "alembic.ini", "upgrade", "head"], + capture_output=True, + text=True, + env={**os.environ, "PRY_DATABASE_URL": f"sqlite:///{temp_data_dir}/migration-test.db"}, + ) + assert result.returncode == 0, f"alembic upgrade failed: {result.stderr}" + + eng = db.get_engine() + inspector = inspect(eng) + tables = inspector.get_table_names() + expected_tables = { + "quality_checks", + "review_items", + "intel_snapshots", + "costing_entries", + "freshness_snapshots", + "structure_snapshots", + "seo_snapshots", + "monitors", + "monitor_runs", + "accounts", + "browser_sessions", + "reports", + "training_datasets", + "pipelines", + "pipeline_runs", + "gdpr_requests", + "agencies", + "agency_clients", + "referral_clicks", + "actors", + "actor_runs", + "llm_usage", + "webhooks", + "x402_receipts", + } + missing = expected_tables - set(tables) + assert not missing, f"Tables missing after migration: {missing}" + + def test_models_have_unique_constraints(temp_data_dir): """Tables that map to JSON stores with _id fields should have unique indexes so that re-importing the same data is idempotent (or at least detectable).""" diff --git a/x402.py b/x402.py index 80b926d..85efbc5 100644 --- a/x402.py +++ b/x402.py @@ -48,6 +48,7 @@ X402_PRICING: dict[str, dict[str, Any]] = { "monitor": {"price_usd": 0.02, "description": "Create scheduled monitor"}, "llm_call": {"price_usd": 0.01, "description": "LLM extraction call"}, "template_execute": {"price_usd": 0.002, "description": "Execute scraper template"}, + "template_batch_execute": {"price_usd": 0.01, "description": "Execute up to 20 template items"}, "browser_automation": {"price_usd": 0.05, "description": "Browser automation"}, "bulk_crawl": {"price_usd": 0.10, "description": "Crawl up to 1000 pages"}, "pdf_extract": {"price_usd": 0.01, "description": "PDF table extraction"}, diff --git a/x402_middleware.py b/x402_middleware.py index 06afc34..d488eec 100644 --- a/x402_middleware.py +++ b/x402_middleware.py @@ -35,6 +35,7 @@ PAID_ENDPOINTS = { "/v1/extract/css": "extract", "/v1/extract/llm": "llm_call", "/v1/templates/execute": "template_execute", + "/v1/templates/batch": "template_batch_execute", "/v1/automate": "browser_automation", "/v1/monitor": "monitor", "/v1/pdf/extract": "pdf_extract",