pryscraper/api.py
cryptorugmunch 9db3f26f95
Some checks failed
CI / lint (pull_request) Failing after 52s
CI / typecheck (pull_request) Failing after 56s
CI / Secret scan (gitleaks) (pull_request) Successful in 32s
CI / Security audit (bandit) (pull_request) Successful in 34s
CI / test (pull_request) Successful in 1m23s
feat(routers): split remaining 174 api.py routes into per-tag routers
- AST-extract all remaining routes from api.py into routers/*.py
- Group routes by OpenAPI tag (AI, Advanced, Agency, ..., x402)
- Keep existing auth/health/scraping/templates routers; add *_api.py for remaining routes of those tags
- Move shared helpers/models/variables with the routes that need them
- Update tests/test_api.py to import vision helpers from routers.vision
- Regenerate openapi.json (186 paths)
- All 497 tests pass; ruff clean
2026-07-03 03:36:58 +02:00

918 lines
19 KiB
Python

"""Pry v3 — Full-stack web intelligence API.
Scrape + Crawl + Automate + Parse + Extract + MCP
Self-hosted, free, better than Firecrawl."""
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 Rug Munch Media LLC
#
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
# Licensed under MIT. See LICENSE.
import json
import time
import uuid
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager, suppress
from typing import Any, cast
import uvicorn
from fastapi import FastAPI, Request, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse, StreamingResponse
from client import close_client
from deps import advanced, automator, extractor, queue, ratelimiter, scraper
from errors import (
PryError,
)
from mcp_production import make_fallback_server, register_all
from mcp_sse import mcp_post_message, mcp_sse_endpoint
from pipeline import get_pipeline
from routers.advanced import router as advanced_router
from routers.agency import router as agency_router
from routers.ai import router as ai_router
from routers.alerts import router as alerts_router
from routers.analysis import router as analysis_router
from routers.auth import router as auth_router
from routers.automation import router as automation_router
from routers.batch import router as batch_router
from routers.circuit_breaker import router as circuit_breaker_router
from routers.commerce import router as commerce_router
from routers.compliance import router as compliance_router
from routers.config import router as config_router
from routers.costing import router as costing_router
from routers.crm import router as crm_router
from routers.dashboard import router as dashboard_router
from routers.email import router as email_router
from routers.enrichment import router as enrichment_router
from routers.execute import router as execute_router
from routers.export import router as export_router
from routers.extraction import router as extraction_router
from routers.freshness import router as freshness_router
from routers.gdpr import router as gdpr_router
from routers.health import router as health_router
from routers.integrations import router as integrations_router
from routers.intelligence import router as intelligence_router
from routers.jobs import router as jobs_router
from routers.marketplace import router as marketplace_router
from routers.monitoring import router as monitoring_router
from routers.parsing import router as parsing_router
from routers.pipeline import router as pipeline_router
from routers.pipelines import router as pipelines_router
from routers.proxy import router as proxy_router
from routers.quality import router as quality_router
from routers.reconciliation import router as reconciliation_router
from routers.recorder import router as recorder_router
from routers.referrals import router as referrals_router
from routers.reports import router as reports_router
from routers.review import router as review_router
from routers.scraping import router as scraping_router
from routers.scraping_api import router as scraping_api_router
from routers.seo import router as seo_router
from routers.sessions import router as sessions_router
from routers.share import router as share_router
from routers.stats import router as stats_router
from routers.structure import router as structure_router
from routers.system import router as system_router
from routers.templates import router as templates_router
from routers.training import router as training_router
from routers.transform import router as transform_router
from routers.untagged import router as untagged_router
from routers.vision import router as vision_router
from routers.webhooks import router as webhooks_router
from routers.x402 import router as x402_router
from settings import settings
from x402_middleware import X402Middleware
# Configure structured (JSON) logging for the whole process. Per
# CONVENTIONS.md Part 5, all Pry logs must be JSON with timestamp, level,
# service, and event. setup_logging() bridges stdlib logging through
# structlog so that any logger.info("event", k=v) becomes a JSON record.
try:
from logging_config import get_logger as _get_logger
from logging_config import setup_logging
setup_logging()
logger = _get_logger(__name__)
except ImportError:
# logging_config not available (e.g., minimal install); use stdlib
pass
# ── Init ──
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""Startup: validate deps. Shutdown: cleanup clients."""
logger.info("pry_startup", version="3.0.0")
if not settings.api_key:
logger.warning("pry_api_key_unset", message="PRY_API_KEY is not set; all protected endpoints are publicly accessible")
get_pipeline() # Initialize pipeline
yield
logger.info("pry_shutdown")
await close_client()
for obj in [scraper, automator, extractor, advanced, queue]:
if hasattr(obj, "close"):
with suppress(Exception):
await obj.close()
app = FastAPI(
title="Pry",
version="3.0.0",
description="Pry open any website. Web scraping + browser automation + document parsing. Self-hosted, free, better than Firecrawl.",
lifespan=lifespan,
)
# ── MCP sub-app ──
# Mounted on /mcp so that streaming SSE endpoints bypass the main app's
# BaseHTTPMiddleware (which buffers response bodies and breaks SSE).
mcp_app = FastAPI(
title="Pry MCP",
version="3.0.0",
description="Model Context Protocol endpoints for Pry.",
)
mcp_app.add_middleware(
CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
)
_mcp_server_instance: Any | None = None
def _get_mcp_server() -> Any:
"""Lazy-initialize the spec-compliant MCP server."""
global _mcp_server_instance
if _mcp_server_instance is None:
_mcp_server_instance = make_fallback_server(base_url=f"http://localhost:{settings.port}")
register_all(_mcp_server_instance)
return _mcp_server_instance
@mcp_app.post("/", tags=["MCP"], summary="MCP JSON-RPC endpoint")
async def mcp_json_rpc(request: Request) -> dict[str, Any]:
"""MCP JSON-RPC endpoint conforming to the 2024-11-05 spec."""
body = await request.json()
server = _get_mcp_server()
return cast(dict[str, Any], await server.handle_request(body))
@mcp_app.get("/health", tags=["MCP"], summary="MCP server health")
async def mcp_health() -> dict[str, Any]:
"""Health check for the MCP server."""
server = _get_mcp_server()
return {
"status": "ok",
"server": "pry-mcp",
"version": "3.0.0",
"protocol_version": "2024-11-05",
"tools_count": len(server.tools),
"resources_count": len(server.resources),
"prompts_count": len(server.prompts),
}
@mcp_app.get("/sse", tags=["MCP"], summary="MCP HTTP+SSE transport")
async def mcp_sse(request: Request) -> StreamingResponse:
"""MCP HTTP+SSE transport endpoint."""
return await mcp_sse_endpoint(request, _get_mcp_server())
@mcp_app.post("/messages/{session_id}", tags=["MCP"], summary="Post MCP JSON-RPC message")
async def mcp_messages(session_id: str, request: Request) -> Response:
"""Receive a JSON-RPC message for an active MCP SSE session."""
return await mcp_post_message(request, session_id, _get_mcp_server())
app.mount("/mcp", mcp_app)
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])
# Gate paid endpoints behind x402 payments. Free endpoints are always public.
app.add_middleware(
X402Middleware,
free_paths=[
"/health",
"/live",
"/ready",
"/docs",
"/openapi.json",
"/dashboard",
"/v1/config",
"/v1/ai/mcp-config",
"/v1/x402/pricing",
"/v1/x402/payment",
"/v1/x402/verify",
"/v1/x402/require-payment",
"/v1/x402/pay",
"/v1/x402/batch-payment",
"/v1/x402/batch-verify",
"/mcp",
"/mcp/health",
"/mcp/sse",
"/mcp/messages/*",
],
)
def add_error_handlers(app: FastAPI) -> None:
@app.exception_handler(PryError)
async def pry_error_handler(request: Request, exc: PryError) -> JSONResponse:
req_id = getattr(request.state, "request_id", uuid.uuid4().hex[:12])
return JSONResponse(
status_code=exc.status_code,
content={
"success": False,
"error": exc.to_dict(),
"request_id": req_id,
},
)
@app.exception_handler(Exception)
async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse:
req_id = getattr(request.state, "request_id", uuid.uuid4().hex[:12])
logger.exception("unhandled_exception", extra={"request_id": req_id})
return JSONResponse(
status_code=500,
content={
"success": False,
"error": {"code": "internal_error", "message": "An unexpected error occurred"},
"request_id": req_id,
},
)
add_error_handlers(app)
# Public paths that don't need authentication
_AUTH_PUBLIC_PATHS: set[str] = {"/health", "/live", "/ready", "/docs", "/openapi.json"}
# Middleware: auth + rate limit + timing + request logging
class PryHttpMiddleware:
"""Pure-ASGI middleware for auth, rate limiting, timing, and logging.
Implemented as ASGI (instead of @app.middleware('http')) so streaming
responses such as Server-Sent Events are not buffered/broken by
BaseHTTPMiddleware.
"""
def __init__(self, app: Any) -> None:
self.app = app
@staticmethod
def _get_header(scope: dict[str, Any], name: bytes) -> str:
for key, value in scope.get("headers", []):
if key.lower() == name.lower():
return cast(bytes, value).decode("latin-1")
return ""
async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
start = time.time()
request_id = self._get_header(scope, b"x-request-id") or uuid.uuid4().hex[:12]
path = scope.get("path", "")
client = scope.get("client")
ip = client[0] if client else "unknown"
# Authentication check
api_key = settings.api_key
if api_key and path not in _AUTH_PUBLIC_PATHS:
auth = self._get_header(scope, b"authorization")
if not (auth.startswith("Bearer ") and auth[7:] == api_key):
body = json.dumps(
{
"success": False,
"error": {"code": "unauthorized", "message": "Invalid or missing API key"},
}
).encode()
await send(
{
"type": "http.response.start",
"status": 401,
"headers": [
(b"content-type", b"application/json"),
(b"content-length", str(len(body)).encode()),
(b"x-request-id", request_id.encode()),
],
}
)
await send({"type": "http.response.body", "body": body})
return
allowed, stats = ratelimiter.check(ip)
if not allowed:
elapsed = time.time() - start
logger.warning(
"rate_limit_blocked",
extra={
"request_id": request_id,
"ip": ip,
"path": path,
"retry_after": stats.get("retry_after", 1),
},
)
body = json.dumps(
{
"success": False,
"error": {"code": "rate_limit_exceeded", "message": "Rate limit exceeded"},
"retry_after": stats.get("retry_after", 1),
}
).encode()
headers = [
(b"content-type", b"application/json"),
(b"content-length", str(len(body)).encode()),
(b"x-ratelimit-limit", str(ratelimiter.default_rpm).encode()),
(b"x-ratelimit-remaining", b"0"),
(b"x-ratelimit-reset", str(stats.get("retry_after", 1)).encode()),
(b"x-request-id", request_id.encode()),
]
await send({"type": "http.response.start", "status": 429, "headers": headers})
await send({"type": "http.response.body", "body": body})
return
logger.info(
"request_start",
extra={
"request_id": request_id,
"method": scope.get("method", "GET"),
"path": path,
"ip": ip,
},
)
ratelimit_limit = str(ratelimiter.default_rpm).encode()
ratelimit_remaining = str(stats.get("remaining", 0)).encode()
ratelimit_reset = str(stats.get("reset_at", 0)).encode()
request_id_bytes = request_id.encode()
async def wrapped_send(message: dict[str, Any]) -> None:
if message["type"] == "http.response.start":
headers = list(message.get("headers", []))
headers.append((b"x-ratelimit-limit", ratelimit_limit))
headers.append((b"x-ratelimit-remaining", ratelimit_remaining))
headers.append((b"x-ratelimit-reset", ratelimit_reset))
headers.append((b"x-request-id", request_id_bytes))
message["headers"] = headers
await send(message)
await self.app(scope, receive, wrapped_send)
elapsed = time.time() - start
logger.info(
"request_end",
extra={
"request_id": request_id,
"method": scope.get("method", "GET"),
"path": path,
"duration": f"{elapsed:.3f}s",
},
)
app.add_middleware(PryHttpMiddleware)
# ── Stats ──
# ── Costing ──
# ── Freshness ──
# ── Parse (Documents) ──
# ── Automate (Browser) ──
# ── Vision — analyze an image via free OpenRouter vision models ───────────────
# Free models default to google/gemma-4-31b-it:free. Accepts:
# - image (base64 data URI or raw base64)
# - url (auto-screenshots, then analyzes)
# - file_path (local PNG/JPG)
# - question (what to ask about the image)
# - model (override default)
# - max_tokens (default 800)
#
# Auto-fallback: if primary model 429s, tries other free models in order.
# ── Sessions ──────────────────────────────────────────────────────────────────
# ── Email Inbox Scraping ──
# ── Integrations ──
# ── Alerts ──
# ── Jobs ──
# ── Config ──
# ── Win 3: WebSocket streaming ──
# ── Win 5: Batch from file + template ──
# ── Win 6: Browser recorder ──
# ── Win 8: Multi-format transform ──
# ── Win 1: Pryfile execution ──
# ── Win 10: Health dashboard HTML ──
# ── Win 1: AI Schema Suggestion ──
# ── Win 2: Circuit Breaker ──
# ── Win 3: Stable Extraction ──
# ── Pipeline hooks ──
# ── Win 4: Data Pipeline ──
# ── Win 5: Shareable Results ──
# ── Compliance ──
# ── GDPR Compliance Portal ──
# ── AI Training Data Pipeline ──
# ── Monitoring (cron-based content change detection with AI judging) ──
# ── Structure Monitor ──
# ── Intelligence (Competitive Intelligence Engine) ──
# ── Quality ──
# ── Reconciliation ──
# ── Auth ──
# (split into routers/auth.py on the api-router-split refactor)
app.include_router(auth_router)
app.include_router(health_router)
app.include_router(scraping_router)
app.include_router(templates_router)
app.include_router(ai_router)
app.include_router(advanced_router)
app.include_router(agency_router)
app.include_router(alerts_router)
app.include_router(analysis_router)
app.include_router(automation_router)
app.include_router(batch_router)
app.include_router(crm_router)
app.include_router(circuit_breaker_router)
app.include_router(commerce_router)
app.include_router(compliance_router)
app.include_router(config_router)
app.include_router(costing_router)
app.include_router(dashboard_router)
app.include_router(email_router)
app.include_router(enrichment_router)
app.include_router(execute_router)
app.include_router(export_router)
app.include_router(extraction_router)
app.include_router(freshness_router)
app.include_router(gdpr_router)
app.include_router(integrations_router)
app.include_router(intelligence_router)
app.include_router(jobs_router)
app.include_router(marketplace_router)
app.include_router(monitoring_router)
app.include_router(parsing_router)
app.include_router(pipeline_router)
app.include_router(pipelines_router)
app.include_router(proxy_router)
app.include_router(quality_router)
app.include_router(reconciliation_router)
app.include_router(recorder_router)
app.include_router(referrals_router)
app.include_router(reports_router)
app.include_router(review_router)
app.include_router(seo_router)
app.include_router(scraping_api_router)
app.include_router(sessions_router)
app.include_router(share_router)
app.include_router(stats_router)
app.include_router(structure_router)
app.include_router(system_router)
app.include_router(training_router)
app.include_router(transform_router)
app.include_router(untagged_router)
app.include_router(vision_router)
app.include_router(webhooks_router)
app.include_router(x402_router)
# ── Review ──
# ── Commerce Sync ──
# ── CRM Sync ──
# ── Pipeline Builder Endpoints ──
# ── Agency ──
# ── SEO Content Change Monitor ──
# ── Reports ──
# ── Enrichment ──
# ── AI Agent Integration ──
# ── Proxy Provider & Affiliate Signup Flow ──
# ── Advanced Scraping (TLS, GraphQL, Schema.org, WebSocket) ──
# ── Advanced: Cookie Warming, PDF, OCR, Dedup, Behavior ──
# ── x402 payment processing ──
# ── Actor marketplace ──
# ── Webhook delivery ──
if __name__ == "__main__":
uvicorn.run(app, host=settings.host, port=settings.port)