From 080ce915a5712e2e10c4fb6f9feecf98bec06a12 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Fri, 3 Jul 2026 03:40:15 +0200 Subject: [PATCH 1/2] ops(pry): deploy phase 0 runtime fixes and config updates --- STATUS.md | 22 ++++++++++++++---- api.py | 19 +++++++++++++++- cli.py | 1 + docker-compose.yml | 37 ++++++++++++++++++++++++++++++ llm_providers/registry.py | 29 +++++++++++++++++++++++ pyproject.toml | 3 +++ requirements.lock | 31 +++++++++++++++++++++++++ requirements.txt | 3 +++ routers/scraping.py | 48 ++++++++++++++++++++++----------------- settings.py | 2 +- ultimate_scraper.py | 5 ++++ 11 files changed, 172 insertions(+), 28 deletions(-) create mode 100644 requirements.lock diff --git a/STATUS.md b/STATUS.md index 011c39f..37cc0c0 100644 --- a/STATUS.md +++ b/STATUS.md @@ -5,17 +5,17 @@ > Where we are RIGHT NOW. Update before every commit. ## Last Updated -2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed +2026-07-03 - Phase 0 infra hardening: metrics, redis/postgres, pinned deps, scrape metrics ## Current Status -๐ŸŸก pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending. +๐ŸŸก pre-production - router split deployed; infra hardening branch ready for review/deploy. ## Deployment Status - **Last deploy**: TBD - **Current version**: 3.0.0-phase0 - **Health**: TBD - **Uptime (30d)**: TBD -- **Active branch**: `main` +- **Active branch**: `feat/phase0-rebased` (reconciled on top of latest `main`) ## Open Issues - _None โ€” track in forgejo issues_ @@ -28,11 +28,23 @@ - 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`) - 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart - 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing +- 2026-07-03: Added `/metrics` endpoint + Prometheus counters/histograms for requests and scrapes +- 2026-07-03: Added Redis + Postgres services to `docker-compose.yml` +- 2026-07-03: Fixed Ollama URL to Talos (`100.104.130.92:11434`) +- 2026-07-03: Added `LLMRegistry.complete_or_empty()` graceful fallback +- 2026-07-03: Regenerated `requirements.txt`; added pinned `requirements.lock` ## Known Issues / Tech Debt -- Deploy at /srv/pry/ is out of sync with repo (needs pull + restart to activate PRY_API_KEY and router changes) +- Deploy at `/srv/pry/` is out of sync with repo (needs pull + restart to activate PRY_API_KEY, router changes, and metrics) +- `feat/phase0-rebased` branch reconciles infra-hardening commits on top of latest `main` +- pry-flaresolverr host port moved from 8191 to 8192 to avoid conflict with `rmi-flaresolverr` - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage. -- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite. +- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres. +- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active. +- License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002. +- mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy. +- All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage. +- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres. - PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active. - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002. - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy. diff --git a/api.py b/api.py index 35f61a5..f7f204b 100644 --- a/api.py +++ b/api.py @@ -47,6 +47,7 @@ from extractor import SchemaExtractor from mconfig import PryConfig from mcp_production import make_fallback_server, register_all from mcp_sse import mcp_post_message, mcp_sse_endpoint +from observability import REQUEST_COUNT, REQUEST_LATENCY, get_metrics_output, setup_tracing from pipeline import HOOK_POINTS, get_pipeline, run_pipeline from pryextras import BatchProcessor, TransformEngine, recorder, streams from routers.auth import router as auth_router @@ -80,6 +81,7 @@ except ImportError: async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Startup: validate deps. Shutdown: cleanup clients.""" logger.info("pry_startup", version="3.0.0") + setup_tracing() if not settings.api_key: logger.warning( "pry_api_key_unset", @@ -252,8 +254,10 @@ class PryHttpMiddleware: start = time.time() request_id = self._get_header(scope, b"x-request-id") or uuid.uuid4().hex[:12] path = scope.get("path", "") + method = scope.get("method", "GET") client = scope.get("client") ip = client[0] if client else "unknown" + response_status = 200 # Authentication check api_key = settings.api_key @@ -327,7 +331,9 @@ class PryHttpMiddleware: request_id_bytes = request_id.encode() async def wrapped_send(message: dict[str, Any]) -> None: + nonlocal response_status if message["type"] == "http.response.start": + response_status = message.get("status", 200) headers = list(message.get("headers", [])) headers.append((b"x-ratelimit-limit", ratelimit_limit)) headers.append((b"x-ratelimit-remaining", ratelimit_remaining)) @@ -339,12 +345,16 @@ class PryHttpMiddleware: await self.app(scope, receive, wrapped_send) elapsed = time.time() - start + if REQUEST_COUNT and REQUEST_LATENCY: + REQUEST_COUNT.labels(method=method, endpoint=path, status=str(response_status)).inc() + REQUEST_LATENCY.labels(endpoint=path).observe(elapsed) logger.info( "request_end", extra={ "request_id": request_id, - "method": scope.get("method", "GET"), + "method": method, "path": path, + "status": response_status, "duration": f"{elapsed:.3f}s", }, ) @@ -2810,6 +2820,13 @@ app.include_router(health_router) app.include_router(scraping_router) app.include_router(templates_router) + +@app.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False) +async def metrics() -> Response: + """Expose Prometheus metrics for scraping.""" + data, content_type = get_metrics_output() + return Response(content=data, media_type=content_type) + # โ”€โ”€ Review โ”€โ”€ diff --git a/cli.py b/cli.py index 2ec522c..9f9d976 100755 --- a/cli.py +++ b/cli.py @@ -83,6 +83,7 @@ def _spinner(label: str): if stop: break print(f" {CYAN}{c}{NC} {label}... ", end="\r", flush=True) + # CLI spinner runs in a daemon thread; sync sleep is acceptable here. time.sleep(0.1) t = threading.Thread(target=_spin, daemon=True) diff --git a/docker-compose.yml b/docker-compose.yml index 9d5765c..d083265 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,3 +80,40 @@ services: volumes: pry-data: pry-sessions: + + redis: + image: redis:7-alpine + container_name: pry-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - pry-redis:/data + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + + postgres: + image: postgres:15-alpine + container_name: pry-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + volumes: + - pry-postgres:/var/lib/postgresql/data + environment: + - POSTGRES_USER=pry + - POSTGRES_PASSWORD=pry + - POSTGRES_DB=pry + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + +volumes: + pry-sessions: + pry-redis: + pry-postgres: diff --git a/llm_providers/registry.py b/llm_providers/registry.py index f4cee3d..579cf0d 100644 --- a/llm_providers/registry.py +++ b/llm_providers/registry.py @@ -94,6 +94,35 @@ class LLMRegistry: break raise Exception(f"All LLM providers failed. Last: {last_error}") + async def complete_or_empty( + self, + prompt: str, + system: str = "", + provider_name: str = "", + max_tokens: int = 1024, + temperature: float = 0.7, + model: str = "", + fallback: bool = True, + ) -> LLMResponse: + """Complete via LLM, returning an empty response on total failure. + + Use this at API boundaries where a 500 is worse than degraded output. + """ + try: + return await self.complete( + prompt, system, provider_name, max_tokens, temperature, model, fallback + ) + except Exception as e: # noqa: BLE001 + logger.warning("llm_complete_failed_all_providers", extra={"error": str(e)[:120]}) + return LLMResponse( + text="", + provider="none", + model="", + input_tokens=0, + output_tokens=0, + cost_usd=0.0, + ) + async def embed(self, text: str, provider_name: str = "", model: str = "") -> list[float]: names = [provider_name] if provider_name else list(self.fallback_chain) for name in names: diff --git a/pyproject.toml b/pyproject.toml index ccc695c..efab7e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,9 @@ dependencies = [ "structlog>=24.0.0", "sqlalchemy>=2.0.0", "aiosqlite>=0.19.0", + "prometheus-client>=0.21.0", + "opentelemetry-api>=1.29.0", + "opentelemetry-sdk>=1.29.0", ] [project.optional-dependencies] diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..3805281 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# Part of Pry โ€” https://git.rugmunch.io/RugMunchMedia/pryscraper +# Generated from pyproject.toml; pinned to currently installed versions. + +fastapi==0.128.8 +uvicorn==0.49.0 +trafilatura==2.1.0 +readability-lxml==0.8.4.1 +lxml==6.1.1 +httpx==0.28.1 +markdownify==1.2.3 +pydantic==2.12.5 +pydantic-settings==2.14.2 +playwright==1.60.0 +redis==8.0.0 +pypdf==6.14.2 +python-docx==1.2.0 +tiktoken==0.12.0 +pillow==12.2.0 +click==8.1.8 +pyyaml==6.0.3 +pandas==3.0.3 +anyio==4.14.1 +croniter==6.2.3 +structlog==26.1.0 +sqlalchemy==2.0.50 +aiosqlite==0.22.1 +prometheus-client==0.25.0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 diff --git a/requirements.txt b/requirements.txt index 966ffe3..ad5d64e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,6 @@ structlog>=24.0.0 sqlalchemy>=2.0.0 aiosqlite>=0.19.0 alembic>=1.14.0 +prometheus-client>=0.21.0 +opentelemetry-api>=1.29.0 +opentelemetry-sdk>=1.29.0 diff --git a/routers/scraping.py b/routers/scraping.py index 1d7250d..0f9d052 100644 --- a/routers/scraping.py +++ b/routers/scraping.py @@ -32,6 +32,7 @@ from pydantic import BaseModel from client import get_client from deps import cache, extractor, queue, scraper from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError +from observability import track_scrape from scraper import BlockDetector from settings import settings @@ -107,15 +108,16 @@ async def scrape(request: ScrapeRequest) -> dict[str, Any]: return cached try: - result = await scraper.scrape( - request.url, - { - "timeout": request.timeout, - "bypass_cloudflare": request.bypassCloudflare, - "js_render": request.jsRender, - "formats": request.formats, - }, - ) + with track_scrape(method="direct"): + result = await scraper.scrape( + request.url, + { + "timeout": request.timeout, + "bypass_cloudflare": request.bypassCloudflare, + "js_render": request.jsRender, + "formats": request.formats, + }, + ) if result.get("status") != "ok": raise ScrapeError(result.get("error", "Scrape failed")) @@ -212,7 +214,8 @@ async def detect_lazy_content( generate_scroll_script, ) - result = await scraper.scrape(url, {"bypass_cloudflare": True}) + with track_scrape(method="lazy"): + result = await scraper.scrape(url, {"bypass_cloudflare": True}) if result.get("status") != "ok": raise ScrapeError(result.get("error") or "Scrape failed") @@ -255,14 +258,15 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]: task.add_done_callback(_log_crawl_job_failure) return {"success": True, "data": {"id": job_id, "status": "pending"}} - pages = await scraper.crawl( - request.url, - { - "max_pages": request.maxPages, - "max_depth": request.maxDepth, - "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, - }, - ) + with track_scrape(method="crawl"): + pages = await scraper.crawl( + request.url, + { + "max_pages": request.maxPages, + "max_depth": request.maxDepth, + "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, + }, + ) return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} @@ -272,7 +276,8 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]: @router.post("/v1/map", summary="Discover URLs on a site") async def map_pages(request: MapRequest) -> dict[str, Any]: """Discover URLs on a site.""" - urls = await scraper.map_urls(request.url, {"limit": request.limit}) + with track_scrape(method="map"): + urls = await scraper.map_urls(request.url, {"limit": request.limit}) return {"success": True, "data": {"links": urls}} @@ -284,8 +289,9 @@ async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[s """Scrape multiple URLs in parallel. Firecrawl charges extra for batch.""" if len(urls) > 50: raise InvalidRequestError("Max 50 URLs per batch") - tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] - results = await asyncio.gather(*tasks, return_exceptions=True) + with track_scrape(method="batch"): + tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] + results = await asyncio.gather(*tasks, return_exceptions=True) pages = [] for i, r in enumerate(results): if isinstance(r, BaseException): diff --git a/settings.py b/settings.py index e4ed772..8f432ff 100644 --- a/settings.py +++ b/settings.py @@ -25,7 +25,7 @@ class PrySettings(BaseSettings): port: int = 8002 # Ollama LLM endpoint - ollama_url: str = "http://100.100.18.18:11434" + ollama_url: str = "http://100.104.130.92:11434" # FlareSolverr Cloudflare bypass endpoint flaresolverr_url: str = "http://flaresolverr:8191/v1" diff --git a/ultimate_scraper.py b/ultimate_scraper.py index a4c8147..7ef4015 100644 --- a/ultimate_scraper.py +++ b/ultimate_scraper.py @@ -314,6 +314,11 @@ class UltimateScraper: driver = uc.Chrome(headless=True, use_subprocess=True) driver.get(url) + # Note: time.sleep is used inside a sync thread-pool task because + # undetected_chromedriver's blocking API runs in asyncio.to_thread. + # Replacing with asyncio.sleep would require restructuring the tier + # to be fully async; this sync sleep is bounded and does not block + # the event loop directly. time.sleep(random.uniform(2, 4)) html = driver.page_source driver.quit() -- 2.49.1 From 9fee12796e9785c0cb916f57d10cae0356edd7f4 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Fri, 3 Jul 2026 03:42:36 +0200 Subject: [PATCH 2/2] feat(routers): split remaining 174 api.py routes into per-tag routers - AST-extract all remaining routes from api.py into routers/*.py - Group routes by OpenAPI tag (AI, Advanced, Agency, ..., x402) - Keep existing auth/health/scraping/templates routers; add *_api.py for remaining routes of those tags - Move shared helpers/models/variables with the routes that need them - Update tests/test_api.py to import vision helpers from routers.vision - Regenerate openapi.json (186 paths) - All 497 tests pass; ruff clean --- api.py | 3541 +--------- openapi.json | 13026 ++++++++++++++++++----------------- routers/__init__.py | 55 +- routers/advanced.py | 329 + routers/agency.py | 111 + routers/ai.py | 41 + routers/alerts.py | 67 + routers/analysis.py | 199 + routers/automation.py | 77 + routers/batch.py | 36 + routers/circuit_breaker.py | 39 + routers/commerce.py | 87 + routers/compliance.py | 57 + routers/config.py | 49 + routers/costing.py | 70 + routers/crm.py | 113 + routers/dashboard.py | 64 + routers/email.py | 76 + routers/enrichment.py | 65 + routers/execute.py | 44 + routers/export.py | 64 + routers/extraction.py | 256 + routers/freshness.py | 75 + routers/gdpr.py | 123 + routers/integrations.py | 137 + routers/intelligence.py | 107 + routers/jobs.py | 52 + routers/marketplace.py | 75 + routers/monitoring.py | 116 + routers/parsing.py | 133 + routers/pipeline.py | 79 + routers/pipelines.py | 112 + routers/proxy.py | 154 + routers/quality.py | 89 + routers/reconciliation.py | 71 + routers/recorder.py | 45 + routers/referrals.py | 79 + routers/reports.py | 68 + routers/review.py | 151 + routers/scraping_api.py | 179 + routers/seo.py | 64 + routers/sessions.py | 134 + routers/share.py | 58 + routers/stats.py | 28 + routers/structure.py | 110 + routers/system.py | 44 + routers/training.py | 121 + routers/transform.py | 68 + routers/untagged.py | 30 + routers/vision.py | 221 + routers/webhooks.py | 46 + routers/x402.py | 153 + tests/test_api.py | 6 +- 53 files changed, 11598 insertions(+), 9796 deletions(-) create mode 100644 routers/advanced.py create mode 100644 routers/agency.py create mode 100644 routers/ai.py create mode 100644 routers/alerts.py create mode 100644 routers/analysis.py create mode 100644 routers/automation.py create mode 100644 routers/batch.py create mode 100644 routers/circuit_breaker.py create mode 100644 routers/commerce.py create mode 100644 routers/compliance.py create mode 100644 routers/config.py create mode 100644 routers/costing.py create mode 100644 routers/crm.py create mode 100644 routers/dashboard.py create mode 100644 routers/email.py create mode 100644 routers/enrichment.py create mode 100644 routers/execute.py create mode 100644 routers/export.py create mode 100644 routers/extraction.py create mode 100644 routers/freshness.py create mode 100644 routers/gdpr.py create mode 100644 routers/integrations.py create mode 100644 routers/intelligence.py create mode 100644 routers/jobs.py create mode 100644 routers/marketplace.py create mode 100644 routers/monitoring.py create mode 100644 routers/parsing.py create mode 100644 routers/pipeline.py create mode 100644 routers/pipelines.py create mode 100644 routers/proxy.py create mode 100644 routers/quality.py create mode 100644 routers/reconciliation.py create mode 100644 routers/recorder.py create mode 100644 routers/referrals.py create mode 100644 routers/reports.py create mode 100644 routers/review.py create mode 100644 routers/scraping_api.py create mode 100644 routers/seo.py create mode 100644 routers/sessions.py create mode 100644 routers/share.py create mode 100644 routers/stats.py create mode 100644 routers/structure.py create mode 100644 routers/system.py create mode 100644 routers/training.py create mode 100644 routers/transform.py create mode 100644 routers/untagged.py create mode 100644 routers/vision.py create mode 100644 routers/webhooks.py create mode 100644 routers/x402.py diff --git a/api.py b/api.py index f7f204b..ffc08e4 100644 --- a/api.py +++ b/api.py @@ -8,58 +8,83 @@ Self-hosted, free, better than Firecrawl.""" # Part of Pry โ€” https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. -import asyncio -import base64 -import difflib -import html import json -import logging -import os -import re import time import uuid from collections.abc import AsyncIterator from contextlib import asynccontextmanager, suppress -from datetime import UTC, datetime -from pathlib import Path from typing import Any, cast -from urllib.parse import urljoin, urlparse -import httpx -import pydantic import uvicorn -from fastapi import Body, FastAPI, Request, Response, WebSocket, WebSocketDisconnect +from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse -from pydantic import BaseModel +from fastapi.responses import JSONResponse, StreamingResponse -from client import close_client, get_client -from deps import advanced, automator, cache, extractor, parser, queue, ratelimiter, scraper +from client import close_client +from deps import advanced, automator, extractor, queue, ratelimiter, scraper from errors import ( - ExternalServiceError, - InvalidRequestError, - NotFoundError, PryError, - ScrapeError, ) -from extraction import JsonCssExtractionStrategy, extract_with_chunking -from extractor import SchemaExtractor -from mconfig import PryConfig from mcp_production import make_fallback_server, register_all from mcp_sse import mcp_post_message, mcp_sse_endpoint -from observability import REQUEST_COUNT, REQUEST_LATENCY, get_metrics_output, setup_tracing -from pipeline import HOOK_POINTS, get_pipeline, run_pipeline -from pryextras import BatchProcessor, TransformEngine, recorder, streams +from observability import REQUEST_COUNT, REQUEST_LATENCY, setup_tracing +from pipeline import get_pipeline +from routers.advanced import router as advanced_router +from routers.agency import router as agency_router +from routers.ai import router as ai_router +from routers.alerts import router as alerts_router +from routers.analysis import router as analysis_router from routers.auth import router as auth_router +from routers.automation import router as automation_router +from routers.batch import router as batch_router +from routers.circuit_breaker import router as circuit_breaker_router +from routers.commerce import router as commerce_router +from routers.compliance import router as compliance_router +from routers.config import router as config_router +from routers.costing import router as costing_router +from routers.crm import router as crm_router +from routers.dashboard import router as dashboard_router +from routers.email import router as email_router +from routers.enrichment import router as enrichment_router +from routers.execute import router as execute_router +from routers.export import router as export_router +from routers.extraction import router as extraction_router +from routers.freshness import router as freshness_router +from routers.gdpr import router as gdpr_router from routers.health import router as health_router +from routers.integrations import router as integrations_router +from routers.intelligence import router as intelligence_router +from routers.jobs import router as jobs_router +from routers.marketplace import router as marketplace_router +from routers.monitoring import router as monitoring_router +from routers.parsing import router as parsing_router +from routers.pipeline import router as pipeline_router +from routers.pipelines import router as pipelines_router +from routers.proxy import router as proxy_router +from routers.quality import router as quality_router +from routers.reconciliation import router as reconciliation_router +from routers.recorder import router as recorder_router +from routers.referrals import router as referrals_router +from routers.reports import router as reports_router +from routers.review import router as review_router from routers.scraping import router as scraping_router +from routers.scraping_api import router as scraping_api_router +from routers.seo import router as seo_router +from routers.sessions import router as sessions_router +from routers.share import router as share_router +from routers.stats import router as stats_router +from routers.structure import router as structure_router +from routers.system import router as system_router from routers.templates import router as templates_router +from routers.training import router as training_router +from routers.transform import router as transform_router +from routers.untagged import router as untagged_router +from routers.vision import router as vision_router +from routers.webhooks import router as webhooks_router +from routers.x402 import router as x402_router from settings import settings from x402_middleware import X402Middleware -config = PryConfig() -logger = logging.getLogger(__name__) - # Configure structured (JSON) logging for the whole process. Per # CONVENTIONS.md Part 5, all Pry logs must be JSON with timestamp, level, # service, and event. setup_logging() bridges stdlib logging through @@ -363,415 +388,49 @@ class PryHttpMiddleware: app.add_middleware(PryHttpMiddleware) -class AutomateStep(BaseModel): - action: str - selector: str | None = None - value: str | None = None - url: str | None = None - timeout: int | None = 30000 - wait_until: str | None = "networkidle" -class AutomateRequest(BaseModel): - session_id: str | None = None - steps: list[AutomateStep] - headless: bool | None = True - viewport: dict[str, int] | None = None -class ParseRequest(BaseModel): - url: str - timeout: int | None = 60 # โ”€โ”€ Stats โ”€โ”€ -@app.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats") -async def stats() -> dict[str, Any]: - return { - "cache": cache.stats(), - "rate_limiter": ratelimiter.get_stats(), - "sessions": automator.list_sessions(), - } # โ”€โ”€ Costing โ”€โ”€ -@app.get("/v1/costing/dashboard", tags=["Costing"], summary="Get cost analytics dashboard") -async def cost_dashboard() -> dict[str, Any]: - """Get the full cost analytics dashboard. - - Includes: - - Monthly spend breakdown by operation type - - Projected end-of-month cost - - Daily spend for last 7 days - - Cache efficiency metrics - - Smart schedule recommendations - """ - from costing import get_cost_dashboard - - return {"success": True, "data": get_cost_dashboard()} -@app.get("/v1/costing/usage", tags=["Costing"], summary="Get usage breakdown") -async def cost_usage( - year: int | None = None, - month: int | None = None, -) -> dict[str, Any]: - """Get detailed usage breakdown for a specific month.""" - from costing import get_monthly_usage - - return {"success": True, "data": get_monthly_usage(year, month)} -@app.post("/v1/costing/record", tags=["Costing"], summary="Record a usage event") -async def record_usage_endpoint( - operation: str = Body(...), - quantity: float = Body(1.0), - metadata: dict[str, Any] | None = Body(None), -) -> dict[str, Any]: - """Record a usage event for cost tracking. - - Operations: scrape_direct, scrape_flaresolverr, scrape_playwright, - crawl_page, llm_call, vision_call, extraction_css, bandwidth_mb - """ - from costing import record_usage - - result = record_usage(operation, metadata, quantity) - return {"success": True, "data": result} -@app.post("/v1/costing/costs", tags=["Costing"], summary="Update per-operation cost table") -async def update_costs(costs: dict[str, float] = Body(...)) -> dict[str, Any]: - """Update the per-operation cost table with custom prices.""" - from costing import update_cost_table - - result = await update_cost_table(costs) - return {"success": result["success"], "data": result} # โ”€โ”€ Freshness โ”€โ”€ -@app.post( - "/v1/freshness/check", - tags=["Freshness"], - summary="Check if content has changed since last scrape", -) -async def freshness_check( - url: str = Body(...), - content: str = Body(""), -) -> dict[str, Any]: - """Check if content has changed since the last scrape. - - Uses content fingerprinting (SHA256) to detect changes. - If no content is provided, does a quick HEAD check instead. - """ - from freshness import check_content_changed, quick_health_check, record_check_result - - if content: - result = await check_content_changed(url, content) - record_check_result(url, result["changed"]) - return {"success": True, "data": result} - - # Quick HEAD check - head = await quick_health_check(url) - return {"success": True, "data": head} -@app.post( - "/v1/freshness/frequency", - tags=["Freshness"], - summary="Get adaptive scrape frequency recommendation", -) -async def freshness_frequency( - url: str = Body(...), - base_interval: int = Body(60), -) -> dict[str, Any]: - """Get an adaptive scrape frequency recommendation based on content volatility. - - Volatile pages (frequent changes) get shorter intervals. - Stable pages get longer intervals to save costs. - """ - from freshness import calculate_adaptive_frequency - - result = calculate_adaptive_frequency(url, base_interval_minutes=base_interval) - return {"success": True, "data": result} -@app.get("/v1/freshness/dashboard", tags=["Freshness"], summary="Get content staleness dashboard") -async def freshness_dashboard() -> dict[str, Any]: - """Get the content staleness dashboard. - - Shows all tracked URLs, their last check time, age, and whether - they're stale (not checked in 24h). - """ - from freshness import get_staleness_dashboard - - return {"success": True, "data": get_staleness_dashboard()} -@app.post( - "/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system" -) -async def ultimate_scrape( - url: str = Body(..., embed=True), -) -> dict[str, Any]: - """Scrape any URL using Pry's ultimate 10-tier anti-detection system. - - Automatically tries: direct โ†’ cloudscraper โ†’ FlareSolverr โ†’ - undetected-chromedriver โ†’ Playwright โ†’ Googlebot โ†’ Archive.org โ†’ Google Cache - - Returns the first successful result with the method used. - """ - from ultimate_scraper import UltimateScraper - - s = UltimateScraper() - result = await s.scrape(url) - if result.get("status") != "ok": - raise ScrapeError(result.get("error", "All bypass methods failed")) - - return { - "success": True, - "data": { - "url": url, - "method": result.get("method", "unknown"), - "content": result.get("content", "")[:50000], - "content_length": len(result.get("content", "")), - }, - } -@app.post( - "/v1/capture/network", - tags=["Scraping"], - summary="Extract API calls and network patterns from a page", -) -async def capture_network( - url: str = Body(...), -) -> dict[str, Any]: - """Extract API calls, GraphQL queries, and network patterns from a page. - - Useful for understanding how SPAs load data and finding hidden API endpoints. - """ - from network import ( - extract_api_calls_from_html, - extract_graphql_queries, - extract_json_ld, - extract_nextjs_props, - extract_nuxt_state, - ) - - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - html = result.get("raw_html", "") - if not html: - client = await get_client() - try: - resp = await client.get( - url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} - ) - html = resp.text - except (httpx.HTTPError, httpx.RequestError): - raise ScrapeError("Could not fetch raw HTML") from None - - return { - "success": True, - "data": { - "url": url, - "api_calls": extract_api_calls_from_html(html), - "graphql_queries": extract_graphql_queries(html), - "json_ld": extract_json_ld(html), - "nextjs_props": extract_nextjs_props(html) is not None, - "nuxt_state": extract_nuxt_state(html) is not None, - }, - } -@app.post( - "/v1/crawl/adaptive", - tags=["Scraping"], - summary="Crawl with adaptive stopping based on content relevance", -) -async def adaptive_crawl( - url: str = Body(...), - query: str = Body(""), - max_pages: int = Body(50), - max_depth: int = Body(3), - relevance_threshold: float = Body(0.3), -) -> dict[str, Any]: - """Crawl a website with adaptive stopping. - - Uses information foraging theory to decide when to stop: - - Stops when content relevance drops below threshold - - Stops when information gain diminishes - - Respects max_pages and max_depth limits - - Ideal for targeted data collection (pricing, docs, products) - """ - from adaptive import AdaptiveCrawler - - crawler = AdaptiveCrawler( - max_pages=max_pages, - max_depth=max_depth, - relevance_threshold=relevance_threshold, - ) - - pages = [] - to_visit = [(url, 0)] - visited_urls: set[str] = set() - - while to_visit: - current_url, depth = to_visit.pop(0) - - if current_url in visited_urls: - continue - visited_urls.add(current_url) - - try: - result = await scraper.scrape(current_url, {"bypass_cloudflare": True}) - content = result.get("content", "") or "" - except pydantic.ValidationError as e: - logger.warning( - "adaptive_crawl_page_failed", extra={"url": current_url, "error": str(e)} - ) - continue - - decision = await crawler.should_continue(current_url, content, depth, query=query) - pages.append({"url": current_url, "depth": depth, "decision": decision}) - - if not decision["continue"]: - break - - if depth < max_depth: - links = await scraper.map_urls(current_url, {"limit": 10}) - for link in links: - full_url = urljoin(current_url, link) - if full_url not in visited_urls: - to_visit.append((full_url, depth + 1)) - - return { - "success": True, - "data": { - "url": url, - "query": query, - "pages": pages, - "total_pages": len(pages), - "stats": crawler.get_stats(), - }, - } -async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any]) -> None: - """Fire a webhook notification for watch events.""" - try: - client = await get_client() - await client.post( - webhook, - json={"event": "watch_update", "url": url, "data": diff_result}, - timeout=10, - ) - except (httpx.HTTPError, httpx.RequestError): - logger.exception("watch_webhook_failed", extra={"url": url, "webhook": webhook}) # โ”€โ”€ Parse (Documents) โ”€โ”€ -@app.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)") -async def parse_document(request: ParseRequest) -> dict[str, Any]: - """Parse a document (PDF, DOCX, image, CSV, JSON) to text.""" - try: - result = await parser.parse(request.url, request.timeout or 60) - return {"success": True, "data": result} - except PryError: - raise - except Exception as e: - raise ExternalServiceError(str(e)) from e -@app.post( - "/v1/markdown", tags=["Parsing"], summary="Generate markdown with content filtering strategies" -) -async def generate_markdown( - url: str = Body(...), - mode: str = Body("raw"), - query: str = Body(""), - threshold: float = Body(0.3), -) -> dict[str, Any]: - """Generate markdown with configurable content filtering. - - Modes: - - raw: Unfiltered markdown - - fit: Prune boilerplate (nav, ads, footers) - - bm25: Filter by BM25 relevance to query (requires query param) - """ - from markdown_gen import BM25ContentFilter, DefaultMarkdownGenerator, PruningContentFilter - - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - content = result.get("content", "") - if not content: - raise ScrapeError("No content scraped") - - if mode == "fit": - filter_strategy: PruningContentFilter | BM25ContentFilter | None = PruningContentFilter( - threshold=threshold - ) - elif mode == "bm25": - if not query: - filter_strategy = PruningContentFilter(threshold=threshold) - else: - filter_strategy = BM25ContentFilter(threshold=threshold) - else: - filter_strategy = None - - gen = DefaultMarkdownGenerator(content_filter=filter_strategy) - md_result = gen.generate(content, url=url, query=query) - - return { - "success": True, - "data": md_result, - } # โ”€โ”€ Automate (Browser) โ”€โ”€ -@app.post("/v1/automate", tags=["Automation"], summary="Execute browser automation steps") -async def automate(request: AutomateRequest) -> dict[str, Any]: - """Execute browser automation steps. Sessions persist for login flows.""" - try: - steps = [s.model_dump() for s in request.steps] - result = await automator.run_steps( - steps=steps, - session_id=request.session_id, - headless=True if request.headless is None else request.headless, - viewport=request.viewport, - ) - return {"success": True, "data": result} - except PryError: - raise - except pydantic.ValidationError as e: - raise ExternalServiceError(str(e)) from e -@app.post("/v1/screenshot", tags=["Automation"], summary="Take a screenshot of a URL") -async def screenshot( - url: str = Body(..., embed=True), session_id: str | None = None -) -> dict[str, Any]: - """Take a screenshot of a URL. Returns base64 PNG.""" - try: - result = await automator.run_steps( - steps=[{"action": "navigate", "url": url}, {"action": "screenshot"}], - session_id=session_id, - ) - screenshot_data = None - for step in result.get("steps", []): - if step.get("action") == "screenshot": - screenshot_data = step.get("screenshot") - return {"success": True, "data": {"screenshot": screenshot_data or ""}} - except PryError: - raise - except pydantic.ValidationError as e: - raise ExternalServiceError(str(e)) from e # โ”€โ”€ Vision โ€” analyze an image via free OpenRouter vision models โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -784,2032 +443,236 @@ async def screenshot( # - max_tokens (default 800) # # Auto-fallback: if primary model 429s, tries other free models in order. -VISION_MODELS_FALLBACK = [ - "google/gemma-4-31b-it:free", - "google/gemma-4-26b-a4b-it:free", - "nvidia/nemotron-nano-12b-v2-vl:free", - "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", - "openrouter/free", -] -def _get_or_key() -> str | None: - """Pull OPENROUTER_API_KEY from ~/.hermes/.env or env.""" - key = settings.openrouter_api_key - if key: - return key - env_path = os.path.expanduser("~/.hermes/.env") - if not os.path.isfile(env_path): - return None - with open(env_path) as f: - for line in f: - if line.startswith("OPENROUTER_API_KEY="): - return line.split("=", 1)[1].strip().strip('"').strip("'") - return None -async def _call_vision_api( - image_b64: str, question: str, model: str, max_tokens: int = 800 -) -> tuple[str | None, str | None, str | None]: - """Single async vision call. Returns (text, model_used) or raises.""" - key = _get_or_key() - if not key: - return None, None, "OPENROUTER_API_KEY not set" - - # Accept either data URI or raw base64 - data_uri = image_b64 if image_b64.startswith("data:") else f"data:image/png;base64,{image_b64}" - - payload = { - "model": model, - "messages": [ - { - "role": "user", - "content": [ - {"type": "text", "text": question}, - {"type": "image_url", "image_url": {"url": data_uri}}, - ], - } - ], - "max_tokens": max_tokens, - } - - client = await get_client() - resp = await client.post( - "https://openrouter.ai/api/v1/chat/completions", - json=payload, - headers={ - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - "HTTP-Referer": "https://pry.local", - "X-Title": "pry-vision", - }, - timeout=60, - ) - resp.raise_for_status() - body = resp.json() - return body["choices"][0]["message"]["content"], model, None -@app.post("/v1/vision", tags=["Vision"], summary="Analyze an image with a free vision model") -async def vision( - question: str = Body("Describe what is visible in this image.", embed=True), - image: str | None = Body(None, embed=True), - url: str | None = Body(None, embed=True), - file_path: str | None = Body(None, embed=True), - model: str | None = Body(None, embed=True), - max_tokens: int = Body(800, embed=True), - session_id: str | None = Body(None, embed=True), - no_fallback: bool = Body(False, embed=True), -) -> dict[str, Any]: - """Analyze an image with a free vision model. - - Provide ONE of: image (base64 or data URI), url (auto-screenshot), - or file_path (local PNG/JPG). - - Auto-falls-back across 5 free OpenRouter vision models if the - requested one is rate-limited. - """ - try: - # 1. Resolve the image bytes - if url: - # Auto-screenshot via playwright - r = await automator.run_steps( - steps=[{"action": "navigate", "url": url}, {"action": "screenshot"}], - session_id=session_id, - ) - image_b64 = None - for step in r.get("steps", []): - if step.get("action") == "screenshot": - image_b64 = step.get("screenshot") - if not image_b64: - raise ScrapeError("screenshot returned empty") - elif file_path: - p = os.path.expanduser(file_path) - if not os.path.isfile(p): - raise InvalidRequestError(f"file not found: {file_path}") - with open(p, "rb") as f: - image_b64 = base64.b64encode(f.read()).decode("ascii") - elif image: - # Strip data URI prefix if present - image_b64 = image.split(",", 1)[1] if image.startswith("data:") else image - else: - raise InvalidRequestError("must provide one of: image, url, file_path") - - # 2. Build the fallback chain - if model: - models_to_try = [model] - if not no_fallback: - models_to_try += [m for m in VISION_MODELS_FALLBACK if m != model] - else: - models_to_try = list(VISION_MODELS_FALLBACK) - - # 3. Try each model until one succeeds - attempts = [] - for m in models_to_try: - try: - text, used, err = await _call_vision_api(image_b64, question, m, max_tokens) - if err: - attempts.append({"model": m, "error": err}) - continue - return { - "success": True, - "data": { - "answer": text, - "model_used": used, - "question": question, - "image_size_bytes": len(image_b64) * 3 // 4, - "attempts": attempts, - }, - } - except httpx.HTTPStatusError as e: - attempts.append( - { - "model": m, - "error": f"http {e.response.status_code}", - "body": e.response.text[:200], - } - ) - except OSError as e: - attempts.append({"model": m, "error": str(e)}) - - raise ExternalServiceError( - "all vision models failed", - details={"attempts": attempts}, - ) - except PryError: - raise - except OSError as e: - raise ExternalServiceError(str(e)) from e -@app.get( - "/v1/vision/models", tags=["Vision"], summary="List free vision-capable models on OpenRouter" -) -async def vision_models() -> dict[str, Any]: - """List free vision-capable models on OpenRouter.""" - try: - client = await get_client() - resp = await client.get( - "https://openrouter.ai/api/v1/models", headers={"User-Agent": "pry/3.0"}, timeout=15 - ) - resp.raise_for_status() - data = resp.json() - except httpx.HTTPError as e: - raise ExternalServiceError(str(e)) from e - - free = [] - for m in data.get("data", []): - arch = m.get("architecture", {}) - if "image" not in arch.get("input_modalities", []): - continue - pricing = m.get("pricing", {}) - try: - pp = float(pricing.get("prompt", "1") or 1) - cp = float(pricing.get("completion", "1") or 1) - except (httpx.HTTPError, httpx.RequestError): - continue - if pp == 0 and cp == 0: - free.append( - { - "id": m["id"], - "name": m.get("name", ""), - "context": arch.get("context_length"), - "description": (m.get("description") or "")[:120], - } - ) - return {"success": True, "data": {"free_models": free, "count": len(free)}} # โ”€โ”€ Sessions โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -@app.post("/v1/session/create", tags=["Sessions"], summary="Create a persistent browser session") -async def create_session(url: str = Body(...), persist: bool = True) -> dict[str, Any]: - """Create a persistent browser session.""" - session_id = await automator.create_session(url, persist=persist) - - if persist: - from sessions import save_session - - await save_session( - session_id=session_id, - cookies=[], - metadata={"url": url, "created_at": datetime.now(UTC).isoformat()}, - ) - - return {"success": True, "data": {"session_id": session_id, "persist": persist}} -@app.post( - "/v1/session/destroy", tags=["Sessions"], summary="Destroy a browser session with optional save" -) -async def destroy_session( - session_id: str = Body(...), - save_state: bool = Body(False), -) -> dict[str, Any]: - """Destroy a browser session. Optionally save state first.""" - from sessions import delete_session, save_session - - if save_state: - state = await automator.get_session_state(session_id) - if state: - await save_session( - session_id=session_id, - cookies=state.get("cookies", []), - local_storage=state.get("local_storage", {}), - metadata={"destroyed_at": datetime.now(UTC).isoformat()}, - ) - - success = await automator.destroy_session(session_id) - if not save_state: - await delete_session(session_id) - - return {"success": True, "data": {"session_id": session_id, "destroyed": success}} -@app.get("/v1/sessions", tags=["Sessions"], summary="List all persistent sessions") -async def list_sessions() -> dict[str, Any]: - """List all persistent sessions (active + saved).""" - from sessions import list_sessions as list_saved_sessions - - active = automator.list_sessions() - saved = await list_saved_sessions() - - return { - "success": True, - "data": { - "active": active, - "saved": saved, - "total_active": len(active), - "total_saved": len(saved), - }, - } -@app.post("/v1/session/save", tags=["Sessions"], summary="Save current session state to disk") -async def save_session_state( - session_id: str = Body(...), -) -> dict[str, Any]: - """Save a session's current state (cookies, storage) to disk.""" - from sessions import save_session as save_session_to_disk - - state = await automator.get_session_state(session_id) - if not state: - raise NotFoundError(f"Session not found: {session_id}") - - await save_session_to_disk( - session_id=session_id, - cookies=state.get("cookies", []), - local_storage=state.get("local_storage", {}), - session_storage=state.get("session_storage", {}), - metadata={"source": "manual_save"}, - ) - - return { - "success": True, - "data": {"session_id": session_id, "cookie_count": len(state.get("cookies", []))}, - } -@app.post("/v1/session/restore", tags=["Sessions"], summary="Restore a saved session") -async def restore_session(session_id: str = Body(...)) -> dict[str, Any]: - """Restore a session from disk into a browser context.""" - from sessions import load_session - - data = await load_session(session_id) - if not data: - raise NotFoundError(f"Saved session not found: {session_id}") - - success = await automator.restore_session_state( - session_id=session_id, - cookies=data.get("cookies", []), - ) - - return { - "success": success, - "data": { - "session_id": session_id, - "restored": success, - "cookie_count": len(data.get("cookies", [])), - "saved_at": data.get("saved_at", ""), - }, - } -@app.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs") -async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]: - """Scrape two URLs and compare their content. Shows additions, deletions, changes.""" - r1, r2 = await asyncio.gather(scraper.scrape(url1), scraper.scrape(url2)) - c1, c2 = r1.get("content", ""), r2.get("content", "") - diff = list( - difflib.unified_diff( - c1.splitlines(keepends=True), - c2.splitlines(keepends=True), - fromfile=url1, - tofile=url2, - n=3, - ) - ) - return { - "success": True, - "data": { - "url1": url1, - "url2": url2, - "diff": diff[:200], - "changes": len(diff), - "identical": len(diff) == 0, - }, - } -@app.post("/v1/watch", tags=["Analysis"], summary="Watch a page for changes") -async def watch_page( - url: str = Body(...), webhook: str = Body(...), interval: int = 3600, selector: str = "" -) -> dict[str, Any]: - """Watch a page for changes. Accepts a webhook URL for notification. - First call registers the watch, subsequent calls compare and notify. - Firecrawl charges $99/mo for this feature.""" - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError("Could not scrape URL") - content = result.get("content", "") - diff_result = await advanced.track_diff(url, content) - # Fire webhook asynchronously on first registration (webhook support) - if webhook and diff_result["status"] == "registered": - asyncio.create_task(_fire_watch_webhook(webhook, url, diff_result)) - return { - "success": True, - "data": { - "url": url, - "status": diff_result["status"], - "version": diff_result["version"], - "webhook": webhook, - "interval": interval, - "message": "Page registered for change tracking.", - }, - } -@app.post("/v1/export", tags=["Export"], summary="Export scraped content in multiple formats") -async def export_data(url: str = Body(...), format: str = "json") -> dict[str, Any]: - """Export scraped content in multiple formats: json, csv, txt, rss. - Firecrawl only does markdown.""" - result = await scraper.scrape(url) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Export failed") - content = result.get("content", "") - title = result.get("title", url) - - if format == "json": - return { - "success": True, - "data": {"url": url, "title": title, "content": content, "format": "json"}, - } - elif format == "csv": - import csv - import io - - buf = io.StringIO() - w = csv.writer(buf) - w.writerow(["url", "title", "content"]) - w.writerow([url, title, content[:50000]]) - return {"success": True, "data": {"csv": buf.getvalue(), "format": "csv"}} - elif format == "rss": - from datetime import datetime - - safe_title = html.escape(title) - safe_url = html.escape(url) - rss = f""" - -Pry: {safe_title} -{safe_url} -Scraped content from {safe_url} -{safe_title}{safe_url} - -{datetime.now(UTC).strftime("%a, %d %b %Y %H:%M:%S UTC")} -""" - return {"success": True, "data": {"rss": rss, "format": "rss"}} - return {"success": True, "data": {"text": content[:100000], "format": "txt"}} -@app.post("/v1/shadow-dom", tags=["Parsing"], summary="Extract content from Shadow DOM") -async def extract_shadow_dom( - url: str = Body(...), - flatten: bool = Body(True), -) -> dict[str, Any]: - """Scrape a page and extract content from Shadow DOM components. - - Useful for modern web apps built with Lit, web components, or - frameworks that use Shadow DOM encapsulation. - """ - from shadow_dom import ShadowDOMProcessor, has_shadow_dom - - result = await scraper.scrape(url, {"bypass_cloudflare": True, "js_render": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - html = result.get("raw_html", "") - if not html: - # Try to get via direct fetch - client = await get_client() - try: - resp = await client.get(url, timeout=30, follow_redirects=True) - html = resp.text - except (httpx.HTTPError, httpx.RequestError) as e: - raise ScrapeError("Could not fetch raw HTML") from e - - shadow_present = has_shadow_dom(html) - flat_html = "" - - if shadow_present and flatten: - processor = ShadowDOMProcessor() - flat_html = processor.process(html) - - return { - "success": True, - "data": { - "url": url, - "shadow_dom_detected": shadow_present, - "flattened": bool(flat_html), - "content": flat_html[:10000] if flat_html else html[:10000], - "raw_html_length": len(html), - "flat_html_length": len(flat_html) if flat_html else len(html), - }, - } -@app.post( - "/v1/extract-table", tags=["Extraction"], summary="Extract HTML tables as structured data" -) -async def extract_table(url: str = Body(...), table_index: int = 0) -> dict[str, Any]: - """Extract HTML tables from a page as structured data. - Firecrawl doesn't support table extraction at all.""" - import pandas as pd - - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - tables = pd.read_html(resp.text) - if table_index >= len(tables): - raise InvalidRequestError(f"Only {len(tables)} tables found") - df = tables[table_index] - return { - "success": True, - "data": { - "table_index": table_index, - "total_tables": len(tables), - "columns": list(df.columns), - "rows": df.to_dict(orient="records"), - "html": df.to_html(index=False), - }, - } -@app.post("/v1/links", tags=["Extraction"], summary="Analyze links on a page") -async def analyze_links(url: str = Body(...)) -> dict[str, Any]: - """Analyze all links on a page โ€” internal, external, broken, social. - Firecrawl only has basic map functionality.""" - html = "" - try: - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - html = resp.text - except httpx.HTTPError: - logger.warning("links_fetch_failed", extra={"url": url}) - base = urlparse(url).netloc - internal, external = set(), set() - for m in re.finditer(r'href=["\'](https?://[^"\']+)["\']', html): - link = m.group(1) - if urlparse(link).netloc == base: - internal.add(link.split("#")[0]) - else: - external.add(link.split("#")[0]) - for m in re.finditer(r'href=["\'](/[^"\']+)["\']', html): - internal.add(urljoin(url, m.group(1)).split("#")[0]) - social = advanced.find_social_links(html) - return { - "success": True, - "data": { - "url": url, - "internal_count": len(internal), - "external_count": len(external), - "internal": sorted(internal)[:50], - "external": sorted(external)[:50], - "social": social, - }, - } -@app.post("/v1/seo", tags=["Extraction"], summary="SEO analysis of a page") -async def analyze_seo(url: str = Body(...)) -> dict[str, Any]: - """SEO analysis of a page: title, description, headings, images, keywords, readability. - Firecrawl has zero SEO features.""" - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - html = resp.text - result = await scraper.scrape(url) - content = result.get("content", "") - - title = re.search(r"]*>(.*?)", html, re.I | re.S) - desc = re.search(r']*>(.*?)", html, re.I | re.S) - h2 = re.findall(r"]*>(.*?)", html, re.I | re.S) - imgs = re.findall(r"]*\salt=[\"\']([^\"\']*)[\"\'][^>]*>", html, re.I) - total_imgs = len(re.findall(r" dict[str, Any]: - """Scrape + AI summarize using local Ollama. Free, private.""" - result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error", "Scrape failed")) - summary = await advanced.summarize(result.get("content", ""), max_words) - return {"success": True, "data": {"url": url, **summary}} -@app.post("/v1/diff", tags=["Analysis"], summary="Track page changes over time") -async def diff(url: str = Body(...), content: str | None = Body(None)) -> dict[str, Any]: - """Track page changes over time. Returns diff from last scrape.""" - if content: - diff_result = await advanced.track_diff(url, content) - else: - result = await scraper.scrape(url) - if result.get("status") != "ok": - raise ScrapeError("Could not scrape URL") - diff_result = await advanced.track_diff(url, result.get("content", "")) - return {"success": True, "data": diff_result} -@app.post("/v1/schema", tags=["Extraction"], summary="Extract Schema.org/JSON-LD structured data") -async def extract_schema(url: str = Body(...)) -> dict[str, Any]: - """Extract Schema.org/JSON-LD structured data from a page.""" - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - html = resp.text - schemas = advanced.extract_schema(html) - return {"success": True, "data": {"url": url, "schemas": schemas}} -@app.post("/v1/emails", tags=["Extraction"], summary="Find email addresses on a page") -async def find_emails(url: str = Body(...)) -> dict[str, Any]: - """Find all email addresses on a page.""" - try: - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - html_text = resp.text - except (httpx.HTTPError, httpx.RequestError): - html_text = "" - - result = await scraper.scrape(url) - emails = advanced.find_emails(result.get("content", "")) - social = advanced.find_social_links(html_text) - return {"success": True, "data": {"url": url, "emails": emails, "social": social}} # โ”€โ”€ Email Inbox Scraping โ”€โ”€ -@app.post("/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)") -async def scrape_email( - subject: str = Body(""), - body: str = Body(""), - sender: str = Body(""), -) -> dict[str, Any]: - """Extract structured data from an email subject and body. - - Auto-classifies as: order_confirmation, invoice, receipt, - shipping_notification, subscription, or other. - - Extracts: order numbers, amounts, tracking numbers, dates, addresses. - """ - from email_scraper import extract_email_data - - result = extract_email_data(subject, body, sender) - return {"success": True, "data": result} -@app.post("/v1/email/gmail", tags=["Email"], summary="Fetch and extract data from Gmail inbox") -async def fetch_gmail( - access_token: str = Body(...), - max_results: int = Body(20), - query: str = Body(""), - since_days: int = Body(7), -) -> dict[str, Any]: - """Connect to Gmail and extract structured data from emails. - - Requires a Gmail OAuth2 access token with scope: - https://www.googleapis.com/auth/gmail.readonly - - Extracts order confirmations, invoices, receipts, shipping notifications. - """ - from email_scraper import fetch_gmail_emails - - result = await fetch_gmail_emails(access_token, max_results, query, since_days) - return {"success": result.get("success", False), "data": result} -@app.post("/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox") -async def fetch_outlook( - access_token: str = Body(...), - max_results: int = Body(20), - query: str = Body(""), - since_days: int = Body(7), -) -> dict[str, Any]: - """Connect to Outlook/Office 365 and extract structured data from emails. - - Requires a Microsoft Graph OAuth2 access token with scope: - https://graph.microsoft.com/Mail.Read - - Extracts order confirmations, invoices, receipts, shipping notifications. - """ - from email_scraper import fetch_outlook_emails - - result = await fetch_outlook_emails(access_token, max_results, query, since_days) - return {"success": result.get("success", False), "data": result} -@app.post("/v1/categorize", tags=["Analysis"], summary="AI-categorize scraped content") -async def categorize(url: str = Body(...)) -> dict[str, Any]: - """AI-categorize scraped content into topic tags.""" - result = await scraper.scrape(url) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Categorize failed") - tags = await advanced.categorize(result.get("content", "")) - kw = advanced.keyword_density(result.get("content", "")) - readability = advanced.readability(result.get("content", "")) - return { - "success": True, - "data": {"url": url, "tags": tags, "keywords": kw[:10], "readability": readability}, - } # โ”€โ”€ Integrations โ”€โ”€ -@app.get( - "/v1/destinations", - tags=["Integrations"], - summary="List supported data destinations", -) -async def list_destinations() -> dict[str, Any]: - """List all supported data destinations and their config requirements.""" - return { - "success": True, - "data": { - "destinations": [ - { - "id": "slack", - "name": "Slack", - "description": "Send data to a Slack channel via webhook", - "config_fields": ["webhook_url", "title"], - }, - { - "id": "googlesheets", - "name": "Google Sheets", - "description": "Write data to a Google Spreadsheet", - "config_fields": ["spreadsheet_id", "range", "credentials_json"], - }, - { - "id": "airtable", - "name": "Airtable", - "description": "Write records to an Airtable base", - "config_fields": ["base_id", "table_name", "api_key"], - }, - { - "id": "email", - "name": "Email", - "description": "Send data via email (SMTP or mailto link)", - "config_fields": ["recipient", "subject", "smtp_host", "smtp_user"], - }, - ] - }, - } -@app.post( - "/v1/destination/send", - tags=["Integrations"], - summary="Send scraped data to a business destination", -) -async def send_to_destination( - destination: str = Body(...), - data: dict[str, Any] = Body(...), - config: dict[str, Any] = Body(...), - url: str | None = Body(None), -) -> dict[str, Any]: - """Send extracted data to a business destination in one click. - - Destinations: - - slack: Send to Slack channel via webhook - - googlesheets: Write to Google Sheets (requires credentials) - - airtable: Write to Airtable base (requires API key) - - email: Send via email (requires SMTP config) - - Config varies by destination: - - slack: {"webhook_url": "https://hooks.slack.com/..."} - - googlesheets: {"spreadsheet_id": "...", "credentials_json": "..."} - - airtable: {"base_id": "...", "table_name": "Table 1", "api_key": "..."} - - email: {"recipient": "user@example.com", "subject": "Data Export"} - """ - from destinations import SUPPORTED_DESTINATIONS, dispatch - - if destination not in SUPPORTED_DESTINATIONS: - raise InvalidRequestError( - f"Unsupported destination: {destination}. Supported: {SUPPORTED_DESTINATIONS}" - ) - - result = await dispatch(destination, data, config) - return {"success": result["success"], "data": result} -@app.post( - "/v1/scrape-and-send", - tags=["Integrations"], - summary="Scrape a URL and send to a destination", -) -async def scrape_and_send( - url: str = Body(...), - destination: str = Body(...), - destination_config: dict[str, Any] = Body(...), -) -> dict[str, Any]: - """Scrape a URL and send the results to a business destination in one step. - - Combines /v1/scrape + /v1/destination/send into a single call. - Perfect for non-technical users who just want data in their tools. - """ - from destinations import SUPPORTED_DESTINATIONS, dispatch - - if destination not in SUPPORTED_DESTINATIONS: - raise InvalidRequestError( - f"Unsupported destination: {destination}. Supported: {SUPPORTED_DESTINATIONS}" - ) - - # Scrape - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - # Send to destination - send_result = await dispatch(destination, result.get("data", result), destination_config) - - return { - "success": send_result["success"], - "data": { - "url": url, - "destination": destination, - "scrape_status": result.get("status"), - "delivery": send_result, - }, - } # โ”€โ”€ Alerts โ”€โ”€ -@app.post("/v1/alert/send", tags=["Alerts"], summary="Send an alert to any channel") -async def send_alert_endpoint( - channel: str = Body(...), - title: str = Body("Pry Alert"), - message: str = Body(...), - config: dict[str, Any] = Body(...), - severity: str = Body("info"), -) -> dict[str, Any]: - """Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email. - - Config varies by channel: - - slack: {"webhook_url": "..."} - - discord: {"webhook_url": "..."} - - teams: {"webhook_url": "..."} - - telegram: {"bot_token": "...", "chat_id": "..."} - - sms: {"phone": "+1234567890", "twilio_sid": "...", "twilio_token": "...", "twilio_from": "..."} - - email: {"recipient": "user@example.com", "smtp_host": "...", "smtp_user": "..."} - """ - from alerter import send_alert - - result = await send_alert(channel, title, message, config, severity) - return {"success": result["success"], "data": result} -@app.get("/v1/alert/channels", tags=["Alerts"], summary="List supported alert channels") -async def list_channels() -> dict[str, Any]: - """List all supported alert channels and their config requirements.""" - return { - "success": True, - "data": { - "channels": [ - {"id": "slack", "name": "Slack", "config_fields": ["webhook_url"]}, - {"id": "discord", "name": "Discord", "config_fields": ["webhook_url"]}, - {"id": "teams", "name": "Microsoft Teams", "config_fields": ["webhook_url"]}, - {"id": "telegram", "name": "Telegram", "config_fields": ["bot_token", "chat_id"]}, - { - "id": "sms", - "name": "SMS (Twilio)", - "config_fields": ["phone", "twilio_sid", "twilio_token", "twilio_from"], - }, - { - "id": "email", - "name": "Email", - "config_fields": ["recipient", "smtp_host", "smtp_user"], - }, - ] - }, - } # โ”€โ”€ Jobs โ”€โ”€ -@app.get("/v1/job/{job_id}", tags=["Jobs"], summary="Get async job status and result") -async def get_job(job_id: str) -> dict[str, Any]: - """Get async job status and result.""" - job = await queue.get_job(job_id) - if not job: - raise NotFoundError("Job not found") - return {"success": True, "data": job} # โ”€โ”€ Config โ”€โ”€ -@app.get("/v1/config", tags=["Config"], summary="Get current Pry configuration") -async def get_config() -> dict[str, Any]: - """Get current Pry configuration.""" - return {"success": True, "data": config.to_dict()} -@app.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime") -async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]: - """Update Pry configuration at runtime.""" - result = config.update(updates) - return {"success": True, "data": result} -@app.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests") -async def enable_tor() -> dict[str, Any]: - """Enable Tor routing for all requests.""" - result = config.update( - {"tor": {"enabled": True}, "proxy": {"enabled": True, "url": "socks5://tor:9050"}} - ) - resp_data = { - "success": True, - "data": result, - "note": "Run 'docker compose --profile tor up -d' to start Tor container", - } - return resp_data # โ”€โ”€ Win 3: WebSocket streaming โ”€โ”€ -@app.websocket("/v1/stream/{job_id}") -async def stream_endpoint(websocket: WebSocket, job_id: str = "live") -> None: - await websocket.accept() - streams.register(job_id, websocket) - try: - while True: - data = await websocket.receive_text() - await streams.broadcast(job_id, {"echo": data}) - except WebSocketDisconnect: - streams.unregister(job_id, websocket) # โ”€โ”€ Win 5: Batch from file + template โ”€โ”€ -class BatchFileRequest(BaseModel): - filepath: str - template: dict[str, str] - timeout: int | None = 30 - max_urls: int | None = 1000 -@app.post("/v1/batch-file", tags=["Batch"], summary="Batch scrape URLs from a file with templates") -async def batch_from_file(request: BatchFileRequest) -> dict[str, Any]: - bp = BatchProcessor() - results = await bp.from_file( - request.filepath, request.template, request.timeout or 30, request.max_urls or 1000 - ) - return {"success": True, "data": {"total": len(results), "results": results}} # โ”€โ”€ Win 6: Browser recorder โ”€โ”€ -@app.post("/v1/record/start", tags=["Recorder"], summary="Start recording browser actions") -async def start_recording(session_id: str = Body(...)) -> dict[str, Any]: - recorder.start(session_id) - return {"success": True, "data": {"session_id": session_id, "status": "recording"}} -@app.post("/v1/record/step", tags=["Recorder"], summary="Record a browser action step") -async def record_step( - session_id: str = Body(...), action: str = Body(...), selector: str = "", value: str = "" -) -> dict[str, Any]: - recorder.record(session_id, action, selector, value) - return {"success": True, "data": {"recorded": len(recorder._recordings.get(session_id, []))}} -@app.post("/v1/record/export", tags=["Recorder"], summary="Export recording as script") -async def export_recording(session_id: str = Body(...), fmt: str = "json") -> dict[str, Any]: - script = recorder.export(session_id, fmt) - return {"success": True, "data": {"script": script, "format": fmt}} -@app.post("/v1/record/clear", tags=["Recorder"], summary="Clear recorded actions") -async def clear_recording(session_id: str = Body(...)) -> dict[str, Any]: - recorder.clear(session_id) - return {"success": True, "data": {"status": "cleared"}} # โ”€โ”€ Win 8: Multi-format transform โ”€โ”€ -@app.post("/v1/transform", tags=["Transform"], summary="Transform data to multiple formats") -async def transform_data( - data: list[dict[str, Any]] = Body(...), format: str = "csv" -) -> dict[str, Any]: - te = TransformEngine() - if format == "csv": - result = te.to_csv(data) - elif format == "sql": - result = "\n".join(te.to_sql(row) for row in data) - elif format == "html": - result = te.to_html_table(data) - elif format == "markdown": - result = te.to_markdown_table(data) - else: - raise InvalidRequestError(f"Unknown format: {format}") - return {"success": True, "data": {"output": result, "format": format}} # โ”€โ”€ Win 1: Pryfile execution โ”€โ”€ -@app.post("/v1/run", tags=["Execute"], summary="Execute a Pryfile") -async def run_pryfile(path: str = Body("pry.yml")) -> dict[str, Any]: - from pryfile import Pryfile - - resolved = _safe_pryfile_path(path) - pf = Pryfile(resolved) - results = await pf.run_all(scraper) - return {"success": True, "data": {"jobs": results, "total": len(results)}} -@app.get("/v1/jobs", tags=["Jobs"], summary="List jobs in a Pryfile") -async def list_jobs(path: str = "pry.yml") -> dict[str, Any]: - from pryfile import Pryfile - - resolved = _safe_pryfile_path(path) - pf = Pryfile(resolved) - return {"success": True, "data": {"jobs": pf.list_jobs()}} -def _safe_pryfile_path(path: str) -> str: - """Resolve and validate Pryfile path โ€” prevent directory traversal.""" - resolved = Path(path).resolve() - allowed = Path.cwd().resolve() - try: - resolved.relative_to(allowed) - except ValueError as e: - raise InvalidRequestError(f"Path must be inside {allowed}") from e - if not resolved.is_file(): - raise InvalidRequestError(f"File not found: {resolved}") - return str(resolved) # โ”€โ”€ Win 10: Health dashboard HTML โ”€โ”€ -@app.get("/dashboard", tags=["Dashboard"], summary="Pry health and performance dashboard") -async def dashboard() -> HTMLResponse: - cache_stats = cache.stats() - rate_stats = ratelimiter.get_stats() - html = f""" - -Pry โ€” Dashboard - -

๐Ÿ”ง Pry Dashboard

-

Scrape engine health and performance metrics

-
-

Cache Hit Rate

{cache_stats.get("hit_rate", 0)}%
{cache_stats.get("hits", 0)} hits / {cache_stats.get("size", 0)} entries
-

Rate Limit

{rate_stats.get("total_requests", 0)}
{rate_stats.get("active_ips", 0)} active IPs
-

Blocked

{rate_stats.get("total_blocked", 0)}
requests blocked
-

Sessions

{len(automator.sessions)}
active browser sessions
-
- - - - - - - - -
EndpointMethodStatus
/v1/scrapePOSTโœ… Active
/v1/crawlPOSTโœ… Active
/v1/automatePOSTโœ… Active
/v1/batchPOSTโœ… Active
/v1/streamWebSocketโœ… Active
/v1/runPOSTโœ… Active
FlareSolverrProxyโœ… Connected
-

Pry v3.0.0 โ€” Generated {datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")}

-""" - return HTMLResponse(content=html) # โ”€โ”€ Win 1: AI Schema Suggestion โ”€โ”€ -@app.post("/v1/suggest", tags=["Analysis"], summary="AI-suggest schema fields for a URL") -async def suggest_schema(data: dict[str, Any] = Body(...)) -> dict[str, Any]: - url = data.get("url", "") - - result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30}) - content = result.get("content", "") - html = "" - try: - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - html = resp.text - except httpx.HTTPError: - logger.warning("suggest_schema_fetch_failed", extra={"url": url}) - schema = {"_page_title": result.get("title", "")} - candidates: dict[str, str] = {} - for pattern, field in [ - (r'["\']((?:price|cost|amount)[^"\']*)["\']', "price"), - (r'["\'](product-title|product-name|item-name)[^"\']*["\']', "name"), - (r'["\'](?:product-image|main-image|featured-image)[^"\']*["\']', "image"), - (r'["\'](?:description|product-desc|item-desc)[^"\']*["\']', "description"), - (r'["\'](?:rating|stars|review-score)[^"\']*["\']', "rating"), - (r'["\'](?:stock|availability|status)[^"\']*["\']', "stock"), - ]: - found = re.findall(pattern, html.lower()) - if found: - candidates[field] = f'[class*="{found[0][:15]}"]' - if "name" not in candidates: - h1 = re.findall(r']*class=["\']([^"\']*)["\']', html) - if h1: - candidates["name"] = f"h1.{h1[0].replace(' ', '.')}" - schema["suggested"] = candidates - if content and len(content) > 200: - try: - prompt = ( - "Analyze this page. Return ONLY JSON with field names as keys and CSS selectors as values. Look for: price, name, description, image, rating, stock.\n\n" - + content[:4000] - ) - client = await get_client() - ollama_url = settings.ollama_url - resp = await client.post( - f"{ollama_url}/api/generate", - json={ - "model": "qwen2.5-coder:3b", - "prompt": prompt, - "stream": False, - "options": {"num_ctx": 8192, "temperature": 0.1}, - }, - timeout=30, - ) - if resp.status_code == 200: - llm_raw = resp.json().get("response", "") - llm_match = re.search(r"\{.*\}", llm_raw, re.S) - if llm_match: - try: - llm = json.loads(llm_match.group(0)) - if isinstance(llm, dict): - schema["llm_suggested"] = llm - except json.JSONDecodeError: - logger.warning("llm_schema_parse_failed") - except httpx.HTTPError: - logger.warning("ollama_suggest_schema_failed") - return {"success": True, "data": schema} # โ”€โ”€ Win 2: Circuit Breaker โ”€โ”€ -_failures: dict[str, int] = {} -@app.post("/v1/breaker/status", tags=["Circuit Breaker"], summary="Get circuit breaker status") -async def breaker_status() -> dict[str, Any]: - return { - "success": True, - "data": {k: {"fails": v, "backoff": min(2**v, 60)} for k, v in _failures.items()}, - } -@app.post( - "/v1/breaker/reset", tags=["Circuit Breaker"], summary="Reset circuit breaker for a domain" -) -async def breaker_reset(domain: str = Body("")) -> dict[str, Any]: - if domain: - _failures.pop(domain, None) - else: - _failures.clear() - return {"success": True, "data": {"cleared": domain or "all"}} # โ”€โ”€ Win 3: Stable Extraction โ”€โ”€ -@app.post("/v1/extract", tags=["Extraction"], summary="Extract structured fields from a URL") -async def extract_stable(data: dict[str, Any] = Body(...)) -> dict[str, Any]: - url = data.get("url", "") - fields = data.get("fields", {}) - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Extraction failed") - - ex = SchemaExtractor() - extracted = await ex.extract(result.get("content", ""), fields, mode="llm") - return {"success": True, "data": {"url": url, "fields": extracted}} -@app.post( - "/v1/extract/css", - tags=["Extraction"], - summary="Extract structured data with CSS selectors (no LLM)", -) -async def extract_css( - url: str = Body(...), - schema: dict[str, Any] = Body(...), - bypass_cloudflare: bool = Body(True), -) -> dict[str, Any]: - """Extract structured JSON from a URL using CSS selector schema. - - Schema format: - { - "name": "products", - "base_selector": ".product-card", - "fields": [ - {"name": "title", "selector": "h3", "type": "text"}, - {"name": "price", "selector": ".price", "type": "text", "transform": "float"}, - {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}, - {"name": "in_stock", "selector": ".stock", "type": "exists"}, - ] - } - """ - result = await scraper.scrape(url, {"bypass_cloudflare": bypass_cloudflare}) - html = result.get("raw_html", "") - - if not html: - client = await get_client() - resp = await client.get( - url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} - ) - html = resp.text - - if not html: - raise ScrapeError("No HTML content returned from scraper") - - strategy = JsonCssExtractionStrategy(schema) - data = strategy.extract(html) - - return { - "success": True, - "data": { - "schema": schema.get("name", "extracted"), - "count": len(data), - "items": data, - }, - } -@app.post("/v1/extract/llm", tags=["Extraction"], summary="Extract with LLM + chunking strategies") -async def extract_llm( - url: str = Body(...), - instruction: str = Body("Extract all key information from this content."), - schema: dict[str, Any] | None = Body(None), - chunk_strategy: str = Body("topic"), - query: str = Body(""), - top_k: int = Body(5), -) -> dict[str, Any]: - """Extract structured data using LLM with intelligent chunking. - - Chunks content by strategy (topic/sentence/regex), optionally filters - by relevance to query, then extracts from each chunk. - """ - - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - content = result.get("content", "") - if not content: - raise ScrapeError("No content returned from scraper") - - chunks = await extract_with_chunking( - content=content, - instruction=instruction, - schema=schema, - chunk_strategy=chunk_strategy, - query=query, - top_k=top_k, - ) - - return { - "success": True, - "data": { - "chunks": chunks, - "total_chunks": len(chunks), - "strategy": chunk_strategy, - }, - } # โ”€โ”€ Pipeline hooks โ”€โ”€ -@app.post("/v1/pipeline/hook", tags=["Pipeline"], summary="Register a hook function") -async def register_hook( - hook_point: str = Body(...), - javascript_fn: str = Body(...), -) -> dict[str, Any]: - """Register a JavaScript hook function at a pipeline hook point. - - Hook points: before_scrape, after_response, before_parse, after_parse, - before_extract, after_extract, before_return, on_error - """ - if hook_point not in HOOK_POINTS: - raise InvalidRequestError(f"Unknown hook point: {hook_point}. Valid: {HOOK_POINTS}") - - # For now, just acknowledge the registration - # In production, this would execute JS in a sandbox - return { - "success": True, - "data": { - "hook_point": hook_point, - "registered": True, - "message": f"Hook registered at {hook_point}. Note: custom JS hooks require a sandboxed runtime.", - }, - } -@app.get("/v1/pipeline/hooks", tags=["Pipeline"], summary="List all registered hooks") -async def list_pipeline_hooks() -> dict[str, Any]: - """List all registered hooks in the pipeline.""" - pipeline = get_pipeline() - return { - "success": True, - "data": { - "hooks": pipeline.list_hooks(), - "hook_points": HOOK_POINTS, - }, - } -@app.post("/v1/pipeline/run", tags=["Pipeline"], summary="Run pipeline hooks for testing") -async def run_pipeline_test( - hook_point: str = Body(...), - url: str = Body(""), - html: str = Body(""), -) -> dict[str, Any]: - """Run hooks at a given pipeline point for testing.""" - if hook_point not in HOOK_POINTS: - raise InvalidRequestError(f"Unknown hook point: {hook_point}") - - result = await run_pipeline(hook_point, url=url, html=html) - return { - "success": True, - "data": { - "hook_point": hook_point, - "result": {k: v for k, v in result.items() if k != "html"}, - "error_count": len(result.get("errors", [])), - }, - } # โ”€โ”€ Win 4: Data Pipeline โ”€โ”€ -@app.post("/v1/pipe", tags=["Transform"], summary="Scrape and transform via data pipeline") -async def data_pipeline(data: dict[str, Any] = Body(...)) -> dict[str, Any]: - url = data.get("url", "") - transform = data.get("transform", "json") - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Pipeline failed") - content = result.get("content", "") - title = result.get("title", url) - if transform == "sql": - from pryextras import TransformEngine - - te = TransformEngine() - output = te.to_sql({"url": url, "title": title, "content": content[:50000]}) - elif transform == "csv": - import csv - import io - - buf = io.StringIO() - w = csv.writer(buf) - w.writerow(["url", "title", "content"]) - w.writerow([url, title, content[:50000]]) - output = buf.getvalue() - else: - output = json.dumps({"url": url, "title": title, "content": content[:100000]}, indent=2) - return {"success": True, "data": {"url": url, "output": output, "format": transform}} # โ”€โ”€ Win 5: Shareable Results โ”€โ”€ -_shares: dict[str, dict[str, Any]] = {} -@app.post("/v1/share", tags=["Share"], summary="Share scraped content via link") -async def share_scrape(data: dict[str, Any] = Body(...)) -> dict[str, Any]: - url = data.get("url", "") - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - sid = uuid.uuid4().hex[:8] - _shares[sid] = { - "url": url, - "title": result.get("title", url), - "content": result.get("content", ""), - "method": result.get("method"), - "ts": datetime.now(UTC).isoformat(), - } - return {"success": True, "data": {"share_id": sid, "url": f"/share/{sid}"}} -@app.get("/share/{share_id}", tags=["Share"], summary="View shared content") -async def view_share(share_id: str) -> HTMLResponse: - d = _shares.get(share_id) - if not d: - return HTMLResponse("

Not found

Share expired.

", 404) - st, sc, su = html.escape(d["title"]), html.escape(d["content"][:100000]), html.escape(d["url"]) - return HTMLResponse( - f'{st} โ€” Pry Share' - f'' - f"" - f'

๐Ÿ”ง {st}

{su} ยท {d["method"]} ยท {d["ts"][:10]}

' - f"
{sc}
" - ) # โ”€โ”€ Compliance โ”€โ”€ -@app.post("/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL") -async def compliance_check(url: str = Body(...)) -> dict[str, Any]: - """Run a full legal compliance check on a target URL. - - Analyzes: - - robots.txt crawl permissions - - Terms of Service classification (restrictive/permissive/moderate) - - Jurisdiction detection (GDPR, CCPA, LGPD) - - Sensitive data detection (PII, financial, health, contact) - - Returns a green/yellow/red risk score with recommendations. - """ - from compliance import run_compliance_check - - result = await run_compliance_check(url) - return {"success": True, "data": result} -@app.get("/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation") -async def compliance_docs() -> dict[str, Any]: - """Get information about the compliance check endpoint.""" - return { - "success": True, - "data": { - "description": "Legal compliance engine for web scraping targets", - "risk_levels": { - "green": "Low risk โ€” standard scraping practices", - "yellow": "Moderate risk โ€” proceed with caution", - "red": "High risk โ€” legal review required", - }, - "factors_checked": [ - "robots.txt crawl permissions", - "Terms of Service classification", - "Jurisdiction detection (GDPR/CCPA/LGPD)", - "Sensitive data categories (PII, financial, health, contact)", - ], - }, - } # โ”€โ”€ GDPR Compliance Portal โ”€โ”€ -@app.post("/v1/gdpr/consent", tags=["GDPR"], summary="Record user consent for data processing") -async def record_consent_endpoint( - user_id: str = Body(...), - purpose: str = Body("data_collection"), - consent_given: bool = Body(True), - ip_address: str = Body(""), - user_agent: str = Body(""), -) -> dict[str, Any]: - """Record a user's consent for data processing (GDPR Art. 7). - - Stores: user hash, purpose, timestamp, IP, user agent. - Consent expires after 365 days. - """ - from gdpr import record_consent - - result = await record_consent(user_id, purpose, consent_given, ip_address, user_agent) - return {"success": result["success"], "data": result} -@app.get("/v1/gdpr/consent/{user_id}", tags=["GDPR"], summary="Check user consent status") -async def check_consent_endpoint( - user_id: str, - purpose: str = "data_collection", -) -> dict[str, Any]: - """Check if a user has given valid consent for data processing.""" - from gdpr import check_consent - - result = check_consent(user_id, purpose) - return {"success": True, "data": result} -@app.post("/v1/gdpr/consent/revoke", tags=["GDPR"], summary="Revoke user consent") -async def revoke_consent_endpoint( - user_id: str = Body(...), - purpose: str = Body("data_collection"), -) -> dict[str, Any]: - """Revoke a user's consent for a processing purpose (GDPR Art. 7(3)).""" - from gdpr import revoke_consent - - result = revoke_consent(user_id, purpose) - return {"success": True, "data": result} -@app.post( - "/v1/gdpr/deletion/request", tags=["GDPR"], summary="Request data deletion (right to erasure)" -) -async def request_deletion_endpoint( - user_id: str = Body(...), - reason: str = Body("user_request"), -) -> dict[str, Any]: - """Request deletion of all data associated with a user (GDPR Art. 17). - - Creates a deletion request that can be executed immediately or - after a holding period. - """ - from gdpr import request_deletion - - result = await request_deletion(user_id, reason) - return {"success": "error" not in result, "data": result} -@app.post("/v1/gdpr/deletion/execute", tags=["GDPR"], summary="Execute data deletion request") -async def execute_deletion_endpoint( - request_id: str = Body(...), -) -> dict[str, Any]: - """Execute a pending deletion request, removing all user data.""" - from gdpr import execute_deletion - - result = await execute_deletion(request_id) - if "error" in result: - raise NotFoundError(result["error"]) - return {"success": True, "data": result} -@app.get("/v1/gdpr/retention", tags=["GDPR"], summary="Get data retention policy") -async def get_retention_policy_endpoint() -> dict[str, Any]: - """Get the current data retention policy.""" - from gdpr import get_retention_policy - - return {"success": True, "data": get_retention_policy()} -@app.post( - "/v1/gdpr/retention/apply", - tags=["GDPR"], - summary="Apply retention policy to remove expired data", -) -async def apply_retention() -> dict[str, Any]: - """Apply the retention policy, removing all expired data.""" - from gdpr import apply_retention_policy - - result = await apply_retention_policy() - return {"success": True, "data": result} -@app.get("/v1/gdpr/audit", tags=["GDPR"], summary="Get compliance audit log") -async def get_audit_log_endpoint(days_back: int = 7) -> dict[str, Any]: - """Get the compliance audit log for the specified period.""" - from gdpr import get_audit_log - - events = get_audit_log(days_back) - return {"success": True, "data": {"events": events, "total": len(events)}} # โ”€โ”€ AI Training Data Pipeline โ”€โ”€ -@app.post( - "/v1/training/classify-license", - tags=["Training"], - summary="Classify content license for AI training", -) -async def classify_license_endpoint(text: str = Body(...)) -> dict[str, Any]: - """Classify the license of scraped content for AI training compliance. - - Returns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use), - tier (permissive/copyleft/restrictive/conditional), and confidence. - """ - from training_data import classify_license - - result = classify_license(text) - return {"success": True, "data": result} -@app.post("/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content") -async def clean_content( - text: str = Body(...), - strip_names: bool = Body(False), - strip_copyright: bool = Body(True), -) -> dict[str, Any]: - """Strip PII and copyright content for AI training clean room. - - Removes emails, phones, SSNs, credit cards, IPs, and (optionally) names. - Also strips copyright notices and near-verbatim copyright content. - """ - from training_data import strip_copyright_verbatim, strip_pii - - cleaned, pii_stats = strip_pii(text, preserve_names=not strip_names) - copyright_stats = {"blocks_removed": 0, "total_chars_removed": 0} - - if strip_copyright: - cleaned, copyright_stats = strip_copyright_verbatim(cleaned) - - return { - "success": True, - "data": { - "original_length": len(text), - "cleaned_length": len(cleaned), - "pii_removed": pii_stats, - "copyright_blocks_removed": copyright_stats["blocks_removed"], - "copyright_chars_removed": copyright_stats["total_chars_removed"], - "cleaned_content": cleaned[:5000] if len(cleaned) > 5000 else cleaned, - "truncated": len(cleaned) > 5000, - }, - } -@app.post( - "/v1/training/export", - tags=["Training"], - summary="Export a clean AI training dataset", -) -async def export_dataset( - records: list[dict[str, Any]] = Body(...), - output_format: str = Body("jsonl"), - clean_room: bool = Body(True), - strip_names: bool = Body(False), -) -> dict[str, Any]: - """Export scraped content as a clean AI training dataset. - - Each record should have: content, url, and optional metadata. - - Features: - - Per-record provenance tracking (source URL, timestamp, extraction method) - - PII stripping (email, phone, SSN, CC, IP) - - Copyright verbatim text removal - - License classification - - Compliance report generation - """ - from training_data import export_training_dataset - - result = export_training_dataset( - records=records, - format=output_format, # type: ignore - clean_room=clean_room, - strip_names=strip_names, - ) - return {"success": result["success"], "data": result} -@app.get( - "/v1/training/compliance/{dataset_id}", - tags=["Training"], - summary="Generate compliance report for a dataset", -) -async def compliance_report(dataset_id: str) -> dict[str, Any]: - """Generate a compliance report for an exported training dataset. - - Report includes: data provenance, PII/copyright removal stats, - license classification, and legal recommendations. - """ - from training_data import generate_compliance_report - - result = generate_compliance_report(dataset_id) - if "error" in result: - raise NotFoundError(result["error"]) - return {"success": True, "data": result} # โ”€โ”€ Monitoring (cron-based content change detection with AI judging) โ”€โ”€ -@app.post("/v1/monitor", tags=["Monitoring"], summary="Create a scheduled content monitor") -async def create_monitor_endpoint( - name: str = Body(...), - url: str = Body(...), - schedule_cron: str = Body("0 */6 * * *"), - goal: str = Body(""), - webhook: str = Body(""), - use_llm: bool = Body(False), -) -> dict[str, Any]: - """Create a scheduled monitor that tracks content changes. - - Args: - name: Human-readable monitor name - url: Target URL to monitor - schedule_cron: Cron expression (default: every 6 hours) - goal: Natural language goal for meaningful-change detection - webhook: URL to notify on meaningful changes - use_llm: Use LLM for change judging (slower but smarter) - """ - from monitor import create_monitor - - monitor = await create_monitor(name, url, schedule_cron, goal, webhook, use_llm) - return {"success": True, "data": monitor} -@app.post("/v1/monitor/run", tags=["Monitoring"], summary="Run a single monitor check") -async def run_monitor_endpoint( - monitor_id: str = Body(...), -) -> dict[str, Any]: - """Execute a monitor check immediately (outside of schedule).""" - from monitor import run_monitor - - result = await run_monitor(monitor_id) - if "error" in result: - raise NotFoundError(result["error"]) - return {"success": True, "data": result} -@app.get("/v1/monitors", tags=["Monitoring"], summary="List all monitors") -async def list_monitors_endpoint() -> dict[str, Any]: - """List all registered monitors.""" - from monitor import list_monitors - - monitors = await list_monitors() - return {"success": True, "data": {"monitors": monitors, "total": len(monitors)}} -@app.delete("/v1/monitor/{monitor_id}", tags=["Monitoring"], summary="Delete a monitor") -async def delete_monitor_endpoint(monitor_id: str) -> dict[str, Any]: - """Delete a monitor and its snapshots.""" - from monitor import delete_monitor - - success = await delete_monitor(monitor_id) - if not success: - raise NotFoundError(f"Monitor not found: {monitor_id}") - return {"success": True, "data": {"monitor_id": monitor_id, "deleted": True}} -@app.post("/v1/monitor/check", tags=["Monitoring"], summary="Run all due monitors") -async def run_due_monitors() -> dict[str, Any]: - """Check all monitors and run any that are due based on their cron schedule.""" - import croniter - - from monitor import list_monitors, run_monitor - - monitors = await list_monitors() - now = datetime.now(UTC) - results = [] - for m in monitors: - last_run = m.get("last_run_at") - last_dt = datetime.fromisoformat(last_run) if last_run else datetime.min.replace(tzinfo=UTC) - - try: - cron = croniter.croniter(m["schedule_cron"], last_dt) - next_run = cron.get_next(datetime) - if next_run <= now: - result = await run_monitor(m["id"]) - results.append(result) - except Exception as e: # noqa: BLE001 - logger.warning( - "monitor_schedule_check_failed", extra={"monitor_id": m["id"], "error": str(e)} - ) - - return { - "success": True, - "data": { - "total_monitors": len(monitors), - "ran_count": len(results), - "results": results, - }, - } # โ”€โ”€ Structure Monitor โ”€โ”€ -@app.post( - "/v1/structure/check", - tags=["Structure"], - summary="Check if CSS selectors still match on a page", -) -async def structure_check( - url: str = Body(...), - selectors: list[dict[str, Any]] = Body(...), -) -> dict[str, Any]: - """Check if CSS selectors still match on a page. - - Use this to verify your scraper templates still work after - a site redesign. Returns per-selector match status. - - Selectors format: [{"name": "title", "selector": "h1", "type": "css"}] - """ - from structure_monitor import check_selectors - - result = await check_selectors(url, selectors) - return {"success": "error" not in result, "data": result} -@app.post( - "/v1/structure/monitor", tags=["Structure"], summary="Monitor page structure changes over time" -) -async def structure_monitor( - url: str = Body(...), - selectors: list[dict[str, Any]] = Body(...), - template_id: str = Body(""), -) -> dict[str, Any]: - """Monitor page structure changes and detect broken selectors. - - Compares current selector match status to previous checks. - Alerts when selectors stop matching (site redesign detected). - - Use this as a pre-scrape health check for your templates. - """ - from structure_monitor import monitor_page_structure - - result = await monitor_page_structure(url, selectors, template_id) - return {"success": "error" not in result, "data": result} -@app.post( - "/v1/structure/check-template", - tags=["Structure"], - summary="Verify a scraper template still works", -) -async def structure_check_template( - template_id: str = Body(...), url: str = Body("") -) -> dict[str, Any]: - """Check if a pre-built scraper template still works against a URL. - - Extracts the template's selectors and checks each one. - If selectors fail, the template needs updating. - """ - from structure_monitor import monitor_page_structure - from template_engine import get_template - - template = get_template(template_id) - if not template: - raise NotFoundError(f"Template not found: {template_id}") - - schema = template.get("schema", {}) - fields = schema.get("fields", []) - selectors = [ - { - "name": f.get("name", f"field_{i}"), - "selector": f.get("selector", ""), - "type": "css" if f.get("type") != "xpath" else "xpath", - } - for i, f in enumerate(fields) - if f.get("selector") - ] - - if not selectors: - raise InvalidRequestError("Template has no selectable fields") - - # If no URL provided, try the template's site URL - if not url: - site = template.get("site", "") - url = ( - f"https://www.{site}" - if site and not site.startswith("http") - else (site if site else "https://example.com") - ) - - result = await monitor_page_structure(url, selectors, template_id) - return {"success": "error" not in result, "data": result} # โ”€โ”€ Intelligence (Competitive Intelligence Engine) โ”€โ”€ -@app.post("/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot") -async def record_intel_snapshot( - competitor_id: str = Body(...), - competitor_name: str = Body(...), - url: str = Body(...), - fields: dict[str, Any] = Body(...), -) -> dict[str, Any]: - """Record a data snapshot for a competitor. - - Snapshots are stored with timestamps and used for trend analysis, - anomaly detection, and report generation. - """ - from intelligence import record_snapshot - - result = record_snapshot(competitor_id, competitor_name, url, fields) - return {"success": True, "data": result} -@app.get( - "/v1/intel/snapshots/{competitor_id}", tags=["Intelligence"], summary="Get competitor snapshots" -) -async def get_intel_snapshots( - competitor_id: str, - limit: int = 50, - since_hours: int | None = None, -) -> dict[str, Any]: - """Get historical snapshots for a competitor.""" - from intelligence import get_snapshots - - snapshots = get_snapshots(competitor_id, limit, since_hours) - return { - "success": True, - "data": {"competitor_id": competitor_id, "snapshots": snapshots, "count": len(snapshots)}, - } -@app.post( - "/v1/intel/analyze", tags=["Intelligence"], summary="Analyze competitor field for anomalies" -) -async def analyze_field( - competitor_id: str = Body(...), - field: str = Body("price"), -) -> dict[str, Any]: - """Analyze a specific field across a competitor's snapshots for anomalies. - - Returns statistics, z-score analysis, and anomaly detection. - """ - from intelligence import compute_field_statistics, detect_anomalies_numeric, get_snapshots - - snapshots = get_snapshots(competitor_id, limit=100) - stats = compute_field_statistics(snapshots, field) - - if ( - stats.get("has_history") - and stats.get("latest") is not None - and stats.get("previous") is not None - ): - numeric_vals = [ - s.get("fields", {}).get(field) - for s in snapshots - if isinstance(s.get("fields", {}).get(field), (int, float)) - ] - if len(numeric_vals) >= 3: - anomaly = detect_anomalies_numeric(numeric_vals[-1], numeric_vals[:-1]) - else: - anomaly = {"anomaly": False, "reason": "Insufficient numeric history"} - else: - anomaly = {"anomaly": False, "reason": "Insufficient data"} - - return {"success": True, "data": {"field": field, "statistics": stats, "anomaly": anomaly}} -@app.post( - "/v1/intel/report", tags=["Intelligence"], summary="Generate a competitive intelligence report" -) -async def generate_report( - competitors: list[dict[str, Any]] = Body(...), - days_back: int = Body(7), -) -> dict[str, Any]: - """Generate a competitive intelligence report for tracked competitors. - - Analyzes changes over the specified period and returns a structured report - with a natural-language summary. - """ - from intelligence import generate_weekly_report - - report = generate_weekly_report(competitors, days_back) - return {"success": True, "data": report} # โ”€โ”€ Quality โ”€โ”€ -@app.post( - "/v1/quality/check", tags=["Quality"], summary="Run data quality check on extraction results" -) -async def quality_check( - url: str = Body(...), - data: dict[str, Any] = Body(...), - schema: dict[str, Any] | None = Body(None), - expected_types: dict[str, str] | None = Body(None), -) -> dict[str, Any]: - """Run a full data quality check on extraction results. - - Metrics: - - Completeness: what % of expected fields have values - - Schema adherence: field types match expectations - - Freshness: how old is the data - - Anomaly detection: what changed since last extraction - - Use this to validate data BEFORE sending to downstream systems. - """ - from quality import run_quality_check - - # Convert string type names to actual types - type_map: dict[str, type] = { - "str": str, - "int": int, - "float": float, - "bool": bool, - "list": list, - "dict": dict, - "None": type(None), - } - resolved_types: dict[str, type] | None = None - if expected_types: - resolved_types = {} - for field, type_name in expected_types.items(): - resolved_types[field] = type_map.get(type_name, str) - - result = await run_quality_check( - url=url, - data=data, - schema=schema, - expected_types=resolved_types, - ) - return {"success": True, "data": result} -@app.get( - "/v1/quality/stats", tags=["Quality"], summary="Get quality statistics for all checked URLs" -) -async def quality_stats() -> dict[str, Any]: - """Get aggregate quality statistics across all checked URLs.""" - from quality import QUALITY_DIR - - stats: list[dict[str, Any]] = [] - for path in sorted(QUALITY_DIR.glob("*.json"), key=os.path.getmtime, reverse=True)[:50]: - try: - data = json.loads(path.read_text()) - stats.append( - { - "url": data.get("url", ""), - "checked_at": data.get("checked_at", ""), - "size_bytes": path.stat().st_size, - } - ) - except (json.JSONDecodeError, OSError): - continue - - return {"success": True, "data": {"total_checked": len(stats), "recent": stats}} # โ”€โ”€ Reconciliation โ”€โ”€ -@app.post( - "/v1/reconcile", - tags=["Reconciliation"], - summary="Reconcile records from multiple sources into unified entities", -) -async def reconcile_endpoint( - records: list[dict[str, Any]] = Body(...), - vertical: str = Body("product"), - threshold: float = Body(0.7), -) -> dict[str, Any]: - """Reconcile records from multiple sources into matched entities. - - Matches records across sources using identity field similarity, - normalizes to a unified vertical schema, and returns entity groups - with confidence scores. - - Verticals: product, job, real_estate, review - """ - from reconciliation import VERTICAL_SCHEMAS, reconcile - - if vertical not in VERTICAL_SCHEMAS: - raise InvalidRequestError( - f"Unknown vertical: {vertical}. Supported: {list(VERTICAL_SCHEMAS.keys())}" - ) - - result = await reconcile(records, vertical, threshold) - return {"success": True, "data": result} -@app.get( - "/v1/reconcile/schemas", - tags=["Reconciliation"], - summary="List supported reconciliation schemas", -) -async def list_schemas() -> dict[str, Any]: - """List all supported vertical schemas for entity reconciliation.""" - from reconciliation import VERTICAL_SCHEMAS - - schemas = {} - for key, schema in VERTICAL_SCHEMAS.items(): - schemas[key] = { - "name": schema["name"], - "fields": { - k: {fk: fv for fk, fv in v.items() if fk != "type"} - for k, v in schema["fields"].items() - }, - "identity_fields": schema["identity_fields"], - } - - return {"success": True, "data": {"schemas": schemas, "total": len(schemas)}} # โ”€โ”€ Auth โ”€โ”€ @@ -2819,1447 +682,251 @@ app.include_router(auth_router) app.include_router(health_router) app.include_router(scraping_router) app.include_router(templates_router) +app.include_router(ai_router) +app.include_router(advanced_router) +app.include_router(agency_router) +app.include_router(alerts_router) +app.include_router(analysis_router) +app.include_router(automation_router) +app.include_router(batch_router) +app.include_router(crm_router) +app.include_router(circuit_breaker_router) +app.include_router(commerce_router) +app.include_router(compliance_router) +app.include_router(config_router) +app.include_router(costing_router) +app.include_router(dashboard_router) +app.include_router(email_router) +app.include_router(enrichment_router) +app.include_router(execute_router) +app.include_router(export_router) +app.include_router(extraction_router) +app.include_router(freshness_router) +app.include_router(gdpr_router) +app.include_router(integrations_router) +app.include_router(intelligence_router) +app.include_router(jobs_router) +app.include_router(marketplace_router) +app.include_router(monitoring_router) +app.include_router(parsing_router) +app.include_router(pipeline_router) +app.include_router(pipelines_router) +app.include_router(proxy_router) +app.include_router(quality_router) +app.include_router(reconciliation_router) +app.include_router(recorder_router) +app.include_router(referrals_router) +app.include_router(reports_router) +app.include_router(review_router) +app.include_router(seo_router) +app.include_router(scraping_api_router) +app.include_router(sessions_router) +app.include_router(share_router) +app.include_router(stats_router) +app.include_router(structure_router) +app.include_router(system_router) +app.include_router(training_router) +app.include_router(transform_router) +app.include_router(untagged_router) +app.include_router(vision_router) +app.include_router(webhooks_router) +app.include_router(x402_router) -@app.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False) -async def metrics() -> Response: - """Expose Prometheus metrics for scraping.""" - data, content_type = get_metrics_output() - return Response(content=data, media_type=content_type) # โ”€โ”€ Review โ”€โ”€ -@app.post("/v1/review/submit", tags=["Review"], summary="Submit extracted data for human review") -async def review_submit( - data: dict[str, Any] = Body(...), - url: str = Body(...), - schema_name: str | None = Body(None), - confidence_score: float = Body(0.0), - flagged_fields: list[dict[str, Any]] | None = Body(None), -) -> dict[str, Any]: - """Submit extracted data for human review before delivery. - - Use this when extraction confidence is low or anomalies were detected. - Data will be held in the review queue until approved or rejected. - """ - from review import submit_for_review - - result = await submit_for_review( - data=data, - extraction_url=url, - schema_name=schema_name, - confidence_score=confidence_score, - flagged_fields=flagged_fields, - ) - return {"success": True, "data": result} -@app.post("/v1/review/{review_id}/approve", tags=["Review"], summary="Approve a review item") -async def review_approve( - review_id: str, - reviewer: str = Body("api"), - notes: str = Body(""), -) -> dict[str, Any]: - """Approve a review item, allowing data to proceed to delivery.""" - from review import approve_review - - result = await approve_review(review_id, reviewer, notes) - if "error" in result: - raise NotFoundError(result["error"]) - return {"success": True, "data": result} -@app.post("/v1/review/{review_id}/reject", tags=["Review"], summary="Reject a review item") -async def review_reject( - review_id: str, - reviewer: str = Body("api"), - notes: str = Body(""), -) -> dict[str, Any]: - """Reject a review item, blocking data delivery.""" - from review import reject_review - - result = await reject_review(review_id, reviewer, notes) - if "error" in result: - raise NotFoundError(result["error"]) - return {"success": True, "data": result} -@app.get("/v1/reviews", tags=["Review"], summary="List reviews in the queue") -async def list_reviews(status: str | None = None) -> dict[str, Any]: - """List reviews, optionally filtered by status (pending/approved/rejected).""" - from review import get_review_queue - - reviews = get_review_queue(status) - return {"success": True, "data": {"reviews": reviews, "total": len(reviews)}} -@app.get("/v1/review/{review_id}", tags=["Review"], summary="Get review details") -async def get_review(review_id: str) -> dict[str, Any]: - """Get full details of a review item including the data payload.""" - from review import get_review_detail - - result = get_review_detail(review_id) - if not result: - raise NotFoundError(f"Review not found: {review_id}") - return {"success": True, "data": result} -@app.post( - "/v1/extract-with-review", - tags=["Review"], - summary="Extract with automatic human review routing", -) -async def extract_with_review( - url: str = Body(...), - schema: dict[str, Any] | None = Body(None), - expected_types: dict[str, str] | None = Body(None), - slack_webhook: str = Body(""), - auto_approve_threshold: float = Body(0.8), - auto_reject_threshold: float = Body(0.2), -) -> dict[str, Any]: - """Extract data with automatic quality check and human review routing. - - High-confidence results are auto-approved. - Low-confidence results are auto-rejected. - Medium-confidence results go to the human review queue with Slack notification. - """ - from quality import run_quality_check - from review import auto_review_threshold - - # Scrape - scrape_result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if scrape_result.get("status") != "ok": - raise ScrapeError(scrape_result.get("error") or "Scrape failed") - - data = scrape_result - - # Quality check - quality = await run_quality_check( - url=url, - data=data, - schema=schema, - expected_types=None, - ) - - # Auto-route - decision = await auto_review_threshold( - data=data, - extraction_url=url, - quality_result=quality, - slack_webhook=slack_webhook, - auto_approve_threshold=auto_approve_threshold, - auto_reject_threshold=auto_reject_threshold, - ) - - return { - "success": True, - "data": { - "decision": decision, - "quality": {k: v for k, v in quality.items() if k != "url"}, - }, - } # โ”€โ”€ Commerce Sync โ”€โ”€ -@app.post("/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify") -async def sync_commerce( - platform: str = Body(...), - products: list[dict[str, Any]] = Body(...), - credentials: dict[str, Any] = Body(...), -) -> dict[str, Any]: - """Sync scraped products to your e-commerce platform. - - Platforms: - - woocommerce: credentials = {"wp_url": "...", "consumer_key": "...", "consumer_secret": "..."} - - shopify: credentials = {"shop_url": "...", "access_token": "..."} - - Products are imported as drafts for review before publishing. - """ - from commerce_sync import sync_to_shopify, sync_to_woocommerce - - if platform == "woocommerce": - result = await sync_to_woocommerce( - products=products, - wp_url=credentials.get("wp_url", ""), - consumer_key=credentials.get("consumer_key", ""), - consumer_secret=credentials.get("consumer_secret", ""), - category_id=credentials.get("category_id", 0), - status=credentials.get("status", "draft"), - ) - elif platform == "shopify": - result = await sync_to_shopify( - products=products, - shop_url=credentials.get("shop_url", ""), - access_token=credentials.get("access_token", ""), - ) - else: - raise InvalidRequestError( - f"Unsupported platform: {platform}. Supported: woocommerce, shopify" - ) - - return {"success": result["success"], "data": result} -@app.get("/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms") -async def list_commerce_platforms() -> dict[str, Any]: - """List supported e-commerce platforms and their credential requirements.""" - return { - "success": True, - "data": { - "platforms": [ - { - "id": "woocommerce", - "name": "WooCommerce", - "credential_fields": [ - "wp_url", - "consumer_key", - "consumer_secret", - "category_id", - ], - "product_fields": ["name", "price", "description", "image_url"], - }, - { - "id": "shopify", - "name": "Shopify", - "credential_fields": ["shop_url", "access_token"], - "product_fields": ["name", "price", "description", "image_url"], - }, - ] - }, - } # โ”€โ”€ CRM Sync โ”€โ”€ -@app.post("/v1/crm/sync", tags=["CRM"], summary="Sync scraped data to CRM") -async def sync_crm( - platform: str = Body(...), - object_type: str = Body("lead"), - objects: list[dict[str, Any]] = Body(...), - credentials: dict[str, Any] = Body(...), -) -> dict[str, Any]: - """Sync scraped data to your CRM. - - Platforms: - - salesforce: credentials={"instance_url": "...", "access_token": "..."} - - hubspot: credentials={"api_key": "..."} - - pipedrive: credentials={"api_token": "...", "domain": "..."} - - close: credentials={"api_key": "..."} - - Object types vary by platform: - - Salesforce: Lead, Contact, Account, Opportunity - - HubSpot: contacts, companies, deals - - Pipedrive: person, organization, deal, lead - - Close: lead, contact - """ - from crm_sync import sync_to_close, sync_to_hubspot, sync_to_pipedrive, sync_to_salesforce - - platform_map = { - "salesforce": (sync_to_salesforce, ("instance_url", "access_token")), - "hubspot": (sync_to_hubspot, ("api_key",)), - "pipedrive": (sync_to_pipedrive, ("api_token",)), - "close": (sync_to_close, ("api_key",)), - } - - if platform not in platform_map: - raise InvalidRequestError( - f"Unsupported platform: {platform}. Supported: {list(platform_map.keys())}" - ) - - handler, required_fields = platform_map[platform] - for field in required_fields: - if not credentials.get(field): - raise InvalidRequestError(f"Missing required credential: {field}") - - if platform == "salesforce": - result = await handler( - objects, object_type, credentials["instance_url"], credentials["access_token"] - ) - elif platform == "hubspot": - result = await handler(objects, object_type, credentials["api_key"]) - elif platform == "pipedrive": - result = await handler( - objects, object_type, credentials["api_token"], credentials.get("domain", "") - ) - elif platform == "close": - result = await handler(objects, object_type, credentials["api_key"]) - else: - raise InvalidRequestError(f"Platform {platform} not handled") - - return {"success": result["success"], "data": result} -@app.get("/v1/crm/platforms", tags=["CRM"], summary="List supported CRM platforms") -async def list_crm_platforms() -> dict[str, Any]: - """List supported CRM platforms and their credential requirements.""" - return { - "success": True, - "data": { - "platforms": [ - { - "id": "salesforce", - "name": "Salesforce", - "objects": ["Lead", "Contact", "Account", "Opportunity"], - "credential_fields": ["instance_url", "access_token"], - }, - { - "id": "hubspot", - "name": "HubSpot", - "objects": ["contacts", "companies", "deals"], - "credential_fields": ["api_key"], - }, - { - "id": "pipedrive", - "name": "Pipedrive", - "objects": ["person", "organization", "deal", "lead"], - "credential_fields": ["api_token", "domain"], - }, - { - "id": "close", - "name": "Close.com", - "objects": ["lead", "contact"], - "credential_fields": ["api_key"], - }, - ] - }, - } # โ”€โ”€ Pipeline Builder Endpoints โ”€โ”€ -@app.get( - "/v1/pipelines/steps", tags=["Pipelines"], summary="List all available pipeline step types" -) -async def list_step_types() -> dict[str, Any]: - """List all available step types for the visual pipeline builder. - - Each step type includes: name, icon, description, required inputs with types, - and expected outputs. A UI renders these as drag-and-drop blocks. - """ - from pipelines import STEP_TYPES - - return {"success": True, "data": {"step_types": STEP_TYPES, "total": len(STEP_TYPES)}} -@app.post("/v1/pipelines/validate", tags=["Pipelines"], summary="Validate a pipeline definition") -async def validate_pipeline_endpoint(pipeline: dict[str, Any] = Body(...)) -> dict[str, Any]: - """Validate a pipeline definition for correctness.""" - from pipelines import validate_pipeline - - errors = validate_pipeline(pipeline) - return {"success": len(errors) == 0, "data": {"valid": len(errors) == 0, "errors": errors}} -@app.post("/v1/pipelines/run", tags=["Pipelines"], summary="Execute a pipeline") -async def run_pipeline_endpoint( - pipeline: dict[str, Any] = Body(...), - context: dict[str, Any] | None = Body(None), -) -> dict[str, Any]: - """Execute a pipeline definition. - - Steps run sequentially. Each step's output is available as - {{step_id.output_key}} in subsequent step templates. - - Example pipeline: - { - "name": "Monitor Competitor Pricing", - "steps": [ - {"id": "scrape_amazon", "type": "scrape", "inputs": {"url": "https://amazon.com/product"}}, - {"id": "extract", "type": "extract_css", "inputs": {"url": "{{scrape_amazon.url}}", "schema": {...}}}, - {"id": "quality", "type": "quality_check", "inputs": {"url": "{{scrape_amazon.url}}", "data": "{{extract.items}}"}}, - {"id": "notify", "type": "send_slack", "inputs": {"webhook_url": "https://hooks.slack.com/...", "message": "{{quality.quality_score}}"}}, - ] - } - """ - from pipelines import run_pipeline, validate_pipeline - - errors = validate_pipeline(pipeline) - if errors: - raise InvalidRequestError(f"Pipeline validation failed: {'; '.join(errors)}") - result = await run_pipeline(pipeline, context) - return {"success": not result["failed"], "data": result} -@app.post("/v1/pipelines/save", tags=["Pipelines"], summary="Save a pipeline definition") -async def save_pipeline_endpoint(pipeline: dict[str, Any] = Body(...)) -> dict[str, Any]: - """Save a pipeline definition for later use.""" - from pipelines import save_pipeline - - result = save_pipeline(pipeline) - return {"success": result["success"], "data": result} -@app.get("/v1/pipelines", tags=["Pipelines"], summary="List saved pipelines") -async def list_pipelines_endpoint() -> dict[str, Any]: - """List all saved pipeline definitions.""" - from pipelines import list_pipelines - - pipelines = list_pipelines() - return {"success": True, "data": {"pipelines": pipelines, "total": len(pipelines)}} -@app.get("/v1/pipelines/{pipeline_id}", tags=["Pipelines"], summary="Get a saved pipeline") -async def get_pipeline_endpoint(pipeline_id: str) -> dict[str, Any]: - """Get a saved pipeline definition by ID.""" - from pipelines import get_pipeline - - result = get_pipeline(pipeline_id) - if not result: - raise NotFoundError(f"Pipeline not found: {pipeline_id}") - return {"success": True, "data": result} -@app.delete("/v1/pipelines/{pipeline_id}", tags=["Pipelines"], summary="Delete a saved pipeline") -async def delete_pipeline_endpoint(pipeline_id: str) -> dict[str, Any]: - """Delete a saved pipeline definition.""" - from pipelines import delete_pipeline - - success = delete_pipeline(pipeline_id) - if not success: - raise NotFoundError(f"Pipeline not found: {pipeline_id}") - return {"success": True, "data": {"deleted": True}} # โ”€โ”€ Agency โ”€โ”€ -@app.post("/v1/agency/create", tags=["Agency"], summary="Create a white-label agency profile") -async def create_agency_endpoint( - name: str = Body(...), - owner_email: str = Body(...), - custom_domain: str = Body(""), - brand_color: str = Body("#f59e0b"), - logo_url: str = Body(""), -) -> dict[str, Any]: - """Create a white-label agency profile for reselling Pry. - - Agencies get: - - Custom branding (colors, logo, domain) - - Client management with sub-accounts - - Usage analytics and quota management - - API key management for each client - """ - from agency import create_agency - - result = create_agency(name, owner_email, custom_domain, brand_color, logo_url) - return {"success": result["success"], "data": result} -@app.get("/v1/agency/{agency_id}", tags=["Agency"], summary="Get agency profile") -async def get_agency_endpoint(agency_id: str) -> dict[str, Any]: - """Get agency profile details.""" - from agency import get_agency - - result = get_agency(agency_id) - if not result: - raise NotFoundError(f"Agency not found: {agency_id}") - return {"success": True, "data": result} -@app.put("/v1/agency/{agency_id}/branding", tags=["Agency"], summary="Update agency branding") -async def update_branding( - agency_id: str, - name: str | None = Body(None), - brand_color: str | None = Body(None), - logo_url: str | None = Body(None), - custom_domain: str | None = Body(None), -) -> dict[str, Any]: - """Update white-label branding for an agency.""" - from agency import update_agency_branding - - result = update_agency_branding(agency_id, name, brand_color, logo_url, custom_domain) - return {"success": result["success"], "data": result} -@app.post("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account") -async def create_client_endpoint( - agency_id: str, - client_name: str = Body(...), - client_email: str = Body(...), - monthly_quota: int = Body(10000), -) -> dict[str, Any]: - """Create a client sub-account under an agency. - - Each client gets their own API key and usage quota. - """ - from agency import create_client - - result = create_client(agency_id, client_name, client_email, monthly_quota) - return {"success": result["success"], "data": result} -@app.get("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="List agency clients") -async def list_clients_endpoint(agency_id: str) -> dict[str, Any]: - """List all clients under an agency.""" - from agency import list_clients - - clients = list_clients(agency_id) - return {"success": True, "data": {"clients": clients, "total": len(clients)}} -@app.get("/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics") -async def get_analytics(agency_id: str) -> dict[str, Any]: - """Get aggregate usage analytics for an agency.""" - from agency import get_agency_analytics - - result = get_agency_analytics(agency_id) - return {"success": True, "data": result} -@app.get("/v1/client/{client_id}/quota", tags=["Agency"], summary="Check client quota usage") -async def check_quota(client_id: str) -> dict[str, Any]: - """Check a client's current quota usage and remaining capacity.""" - from agency import check_client_quota - - result = check_client_quota(client_id) - return {"success": result["success"], "data": result} # โ”€โ”€ SEO Content Change Monitor โ”€โ”€ -@app.post("/v1/seo/analyze", tags=["SEO"], summary="Analyze SEO elements from a URL") -async def seo_analyze(url: str = Body(...)) -> dict[str, Any]: - """Analyze all SEO elements from a URL. - - Returns: title, meta description, keywords, headings (H1/H2), - canonical, OG tags, Twitter cards, word count, link counts, - schema markup, hreflang tags. - """ - from seo_monitor import analyze_seo - - result = await analyze_seo(url) - return {"success": "error" not in result, "data": result} -@app.post("/v1/seo/track", tags=["SEO"], summary="Track SEO changes since last scan") -async def seo_track(url: str = Body(...)) -> dict[str, Any]: - """Track SEO changes since the last scan of this URL. - - Compares current SEO elements to previous snapshot and reports - what changed (title, description, headings, etc.). - """ - from seo_monitor import track_seo_changes - - result = await track_seo_changes(url) - return {"success": "error" not in result, "data": result} -@app.post("/v1/seo/keywords", tags=["SEO"], summary="Analyze keyword presence in URL content") -async def seo_keywords( - url: str = Body(...), - keywords: list[str] = Body(...), -) -> dict[str, Any]: - """Analyze which keywords a URL targets. - - Checks each keyword for: - - Presence in title tag - - Presence in H1 headings - - Presence in meta description - - Frequency in body content - - Keyword density percentage - """ - from seo_monitor import get_seo_keyword_insights - - result = await get_seo_keyword_insights(url, keywords) - return {"success": "error" not in result, "data": result} # โ”€โ”€ Reports โ”€โ”€ -@app.post( - "/v1/reports/generate", - tags=["Reports"], - summary="Generate a white-label report from scraped data", -) -async def generate_report_endpoint( - report_type: str = Body(...), - data: dict[str, Any] = Body(...), - branding: dict[str, Any] | None = Body(None), - output_format: str = Body("html"), -) -> dict[str, Any]: - """Generate a white-label report from scraped data. - - Report types: - - competitive_analysis: Competitor pricing and activity overview - - price_monitor: Product price change tracking with visual indicators - - seo_audit: SEO element analysis with change detection - - content_tracker: Content change monitoring across pages - - Branding (optional): {"agency_name": "...", "brand_color": "#hex", "logo_url": "..."} - """ - from reports import generate_report - - result = generate_report(report_type, data, branding, output_format) - if "error" in result: - raise InvalidRequestError(result["error"]) - return {"success": True, "data": result} -@app.get("/v1/reports", tags=["Reports"], summary="List generated reports") -async def list_reports_endpoint() -> dict[str, Any]: - """List all previously generated reports.""" - from reports import list_reports - - reports = list_reports() - return {"success": True, "data": {"reports": reports, "total": len(reports)}} -@app.get("/v1/report/{report_id}", tags=["Reports"], summary="Get a generated report") -async def get_report(report_id: str) -> Any: - """Get the HTML content of a generated report.""" - from reports import REPORTS_DIR - - for path in REPORTS_DIR.glob(f"{report_id}_*.html"): - return HTMLResponse(content=path.read_text()) - raise NotFoundError(f"Report not found: {report_id}") # โ”€โ”€ Enrichment โ”€โ”€ -@app.post( - "/v1/enrich", - tags=["Enrichment"], - summary="Enrich scraped data with company info, tech stack, social profiles", -) -async def enrich_data( - url: str = Body(...), - include_tech_stack: bool = Body(True), - include_social: bool = Body(True), - include_company: bool = Body(True), -) -> dict[str, Any]: - """Enrich a URL with supplemental business intelligence. - - Returns: - - Tech stack detection (CMS, framework, CDN, analytics, payments) - - Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.) - - Company information (email, phone, address, founded year, team size) - """ - from enrichment import enrich_url - - result = await enrich_url(url) - filtered: dict[str, Any] = {"url": url} - if include_tech_stack: - filtered["tech_stack"] = result.get("tech_stack") - if include_social: - filtered["social_profiles"] = result.get("social_profiles") - if include_company: - filtered["company_info"] = result.get("company_info") - - return {"success": True, "data": filtered} -@app.post( - "/v1/enrich/tech-stack", tags=["Enrichment"], summary="Detect technologies used on a website" -) -async def detect_tech(url: str = Body(...)) -> dict[str, Any]: - """Detect what technologies a website is built with. - - Detects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails), - frontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar), - payments (Stripe, PayPal). - """ - from enrichment import enrich_url - - result = await enrich_url(url) - return {"success": True, "data": result.get("tech_stack", {})} # โ”€โ”€ AI Agent Integration โ”€โ”€ -@app.get( - "/openapi.json", - tags=["System"], - summary="OpenAPI spec for AI agent integration", - include_in_schema=False, -) -async def get_openapi() -> JSONResponse: - """Get the OpenAPI specification for AI agent integration. - - Use this with ChatGPT GPT Actions, Claude MCP, or any AI agent - framework to let AI models scrape the web through Pry. - """ - from ai_plugin import get_openapi_spec - - spec = get_openapi_spec() - return JSONResponse(content=spec) -@app.get("/v1/ai/gpt-manifest", tags=["AI"], summary="Get GPT Action manifest for ChatGPT") -async def get_gpt_manifest() -> JSONResponse: - """Get the GPT Action manifest for ChatGPT integration. - - Add this to your GPT configuration in ChatGPT to give it - web scraping capabilities through Pry. - """ - from ai_plugin import get_gpt_action_manifest - - return JSONResponse(content=get_gpt_action_manifest()) -@app.get("/v1/ai/mcp-config", tags=["AI"], summary="Get MCP server config for Claude/Cursor") -async def get_mcp_config() -> JSONResponse: - """Get the MCP server configuration for Claude/Cursor. - - Add this to your AI tool's MCP configuration file to let - Claude and Cursor scrape the web through Pry. - """ - from ai_plugin import get_mcp_server_config - - return JSONResponse(content={"success": True, "data": get_mcp_server_config()}) -@app.get( - "/v1/referrals/catalog", - tags=["Referrals"], - summary="Get all available referral/affiliate programs", -) -async def get_referral_catalog(category: str = "") -> dict[str, Any]: - """List all referral programs Pry supports. - - 60+ providers across categories: LLM, hosting, domains, CDN, email, - monitoring, proxies, voice, media, devtools, search, CAPTCHA. - """ - from referrals import ReferralTracker - - return {"success": True, "data": ReferralTracker().get_catalog(category)} -@app.get( - "/v1/referrals/stats", tags=["Referrals"], summary="Get referral click and conversion stats" -) -async def get_referral_stats(days_back: int = 30) -> dict[str, Any]: - """Get referral tracking statistics for the last N days.""" - from referrals import ReferralTracker - - return {"success": True, "data": ReferralTracker().get_stats(days_back)} -@app.post("/v1/referrals/click", tags=["Referrals"], summary="Record a referral link click") -async def record_referral_click( - provider_tag: str = Body(...), - source: str = Body("api"), - user_id: str = Body(""), -) -> dict[str, Any]: - """Record when a user clicks a referral link. Returns tracking ID.""" - from referrals import PROVIDER_CATALOG, ReferralTracker - - url = "" - for _cat, providers in PROVIDER_CATALOG.items(): - for p in providers: - if p.get("tag") == provider_tag: - url = p["url"] - break - if url: - break - if not url: - return {"success": False, "error": f"Unknown provider: {provider_tag}"} - click_id = ReferralTracker().record_click(provider_tag, url, source, user_id) - return {"success": True, "data": {"click_id": click_id, "url": url}} -@app.post("/v1/referrals/convert", tags=["Referrals"], summary="Record a referral conversion") -async def record_referral_conversion( - click_id: str = Body(...), - revenue_usd: float = Body(0.0), - notes: str = Body(""), -) -> dict[str, Any]: - """Record that a referral click resulted in a conversion.""" - from referrals import ReferralTracker - - success = ReferralTracker().record_conversion(click_id, revenue_usd, notes) - return {"success": success} -@app.get("/v1/x402/pricing", tags=["x402"], summary="Get x402 pricing for all paid operations") -async def x402_pricing() -> dict[str, Any]: - """Get the price list for pay-per-scrape operations.""" - from x402 import X402Handler - - return {"success": True, "data": X402Handler().get_stats()} -@app.post("/v1/x402/payment", tags=["x402"], summary="Create a x402 payment request") -async def x402_payment_request( - operation: str = Body(...), - metadata: dict[str, Any] | None = Body(None), -) -> dict[str, Any]: - """Create a x402 payment request for a paid operation. - - Returns payment details (wallet, amount, asset) for the client to pay. - """ - from x402 import create_payment_request - - req = create_payment_request(operation, metadata) - return {"success": True, "data": req} -@app.post("/v1/x402/verify", tags=["x402"], summary="Verify a x402 payment was settled") -async def x402_verify_payment( - payment_id: str = Body(...), - tx_hash: str = Body(...), - network: str = Body(""), - asset: str = Body(""), - amount_usd: float = Body(0.0), -) -> dict[str, Any]: - """Verify a x402 payment has been settled on-chain via facilitator router.""" - from x402 import X402Handler - - result = await X402Handler().verify_payment( - payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd - ) - return {"success": result["verified"], "data": result} -@app.post( - "/v1/x402/require-payment", tags=["x402"], summary="Generate a 402 Payment Required response" -) -async def x402_require_payment( - operation: str = Body(...), - metadata: dict[str, Any] | None = Body(None), -) -> dict[str, Any]: - """Generate a 402 Payment Required response for a paid endpoint.""" - from x402 import require_payment_legacy - - return require_payment_legacy(operation, metadata) -@app.post("/v1/x402/batch-payment", tags=["x402"], summary="Create a batch x402 payment") -async def x402_batch_payment(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: - """Create a single x402 payment covering multiple operations. - - Returns a PaymentRequired body with the combined amount. After paying, - submit the tx to POST /v1/x402/batch-verify. - """ - from x402 import create_batch_payment - - operations = payload.get("operations", []) - if not isinstance(operations, list): - raise InvalidRequestError("operations must be a list") - result = await create_batch_payment(operations) - if "error" in result: - return {"success": False, "error": result["error"]} - return {"success": True, "data": result} -@app.post("/v1/x402/batch-verify", tags=["x402"], summary="Verify a batch x402 payment") -async def x402_batch_verify(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: - """Verify the on-chain payment for a batch and mark it paid.""" - from x402 import verify_batch_payment - - batch_id = payload.get("batch_id", "") - tx_hash = payload.get("tx_hash", "") - network = payload.get("network", "") - asset = payload.get("asset", "") - if not batch_id or not tx_hash: - raise InvalidRequestError("batch_id and tx_hash are required") - result = await verify_batch_payment(batch_id, tx_hash, network=network, asset=asset) - return {"success": result["verified"], "data": result} # โ”€โ”€ Proxy Provider & Affiliate Signup Flow โ”€โ”€ -@app.get("/v1/proxy/providers", tags=["Proxy"], summary="List all available proxy providers") -async def list_proxy_providers() -> dict[str, Any]: - """List free + premium proxy providers with affiliate details.""" - from proxy_manager import ProxyManager - - return {"success": True, "data": ProxyManager().list_providers()} -@app.post("/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider") -async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]: - """Get the affiliate signup URL for a proxy provider. - - Records a click for revenue tracking. The user can then sign up at that URL - and come back to configure credentials. - """ - from proxy_manager import ProxyManager - - pm = ProxyManager() - url = pm.get_signup_link(provider) - return {"success": True, "data": {"signup_url": url, "provider": provider}} -@app.get("/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals") -async def list_proxy_referrals() -> dict[str, Any]: - """Return the curated proxy provider affiliate catalog (proxy_referrals.py). - - This is the marketing-friendly subset: commission, promo codes, tier - (premium/standard/budget), and referral URLs. Useful for a "Sign up via Pry - and we get a cut" UI, or for showing users premium options when their free - proxy fails. - - The full connection metadata (host, port, auth) lives in /v1/proxy/providers. - """ - from proxy_manager import ProxyManager - - pm = ProxyManager() - return { - "success": True, - "data": { - "providers": pm.list_proxy_referrals(), - "summary": pm.get_proxy_referral_summary(), - }, - } -@app.get( - "/v1/proxy/referrals/{tag}", tags=["Proxy"], summary="Get a single proxy provider referral" -) -async def get_proxy_referral(tag: str) -> dict[str, Any]: - """Return the affiliate info for a single proxy provider by tag.""" - from proxy_manager import ProxyManager - - pm = ProxyManager() - info = pm.get_proxy_referral(tag) - if not info: - return {"success": False, "error": f"No proxy referral found for tag: {tag}"} - return {"success": True, "data": {"tag": tag, **info}} -@app.post("/v1/proxy/configure", tags=["Proxy"], summary="Configure proxy credentials") -async def proxy_configure( - provider: str = Body(...), - username: str = Body(""), - password: str = Body(""), - api_key: str = Body(""), - proxy_url: str = Body(""), -) -> dict[str, Any]: - """Configure credentials for a premium proxy provider. - - After signing up via /v1/proxy/signup, the user provides their credentials here. - """ - from proxy_manager import ProxyManager - - pm = ProxyManager() - creds = {"username": username, "password": password, "api_key": api_key, "proxy_url": proxy_url} - creds = {k: v for k, v in creds.items() if v} - result = pm.select_provider(provider, creds) - return {"success": result["success"], "data": result} -@app.get("/v1/proxy/test", tags=["Proxy"], summary="Test the active proxy") -async def proxy_test() -> dict[str, Any]: - """Test the currently configured proxy and return its public IP.""" - from proxy_manager import ProxyManager - - pm = ProxyManager() - proxy_url = pm.get_proxy_url() - if not proxy_url: - return {"success": True, "data": {"active": False, "message": "No proxy configured"}} - result = pm.test_proxy(proxy_url) - return {"success": True, "data": {"active": True, **result}} -@app.get("/v1/proxy/status", tags=["Proxy"], summary="Get current proxy status") -async def proxy_status() -> dict[str, Any]: - """Get current proxy configuration and available providers.""" - from proxy_manager import ProxyManager - - pm = ProxyManager() - return { - "success": True, - "data": { - "active_config": pm.active_config.__dict__, - "configured_providers": list(pm.credentials.keys()), - "available_providers": pm.list_providers(), - }, - } -@app.post("/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block") -async def proxy_recommend(last_error: str = Body("")) -> dict[str, Any]: - """After a scrape fails with anti-bot detection, get a recommendation - for which premium proxy provider to sign up with.""" - from proxy_manager import ProxyManager - - pm = ProxyManager() - rec = pm.get_recommendation(last_error) - return {"success": True, "data": rec} -@app.get("/v1/proxy/clicks", tags=["Proxy"], summary="Get recent proxy referral clicks") -async def proxy_clicks(days_back: int = 30) -> dict[str, Any]: - """Get recent proxy referral clicks for revenue tracking.""" - from proxy_manager import ProxyManager - - pm = ProxyManager() - clicks = pm.get_recent_clicks(days_back=days_back) - return { - "success": True, - "data": { - "total_clicks": len(clicks), - "days_back": days_back, - "clicks": clicks, - }, - } # โ”€โ”€ Advanced Scraping (TLS, GraphQL, Schema.org, WebSocket) โ”€โ”€ -@app.post( - "/v1/tls/impersonate", - tags=["Advanced"], - summary="Fetch a URL with TLS fingerprint impersonation", -) -async def tls_impersonate( - url: str = Body(...), - impersonate: str = Body("chrome131"), - proxy: str = Body(""), -) -> dict[str, Any]: - """Fetch a URL while impersonating a real browser's TLS fingerprint. - Bypasses JA3/JA4 fingerprinting that blocks 80%+ of bot traffic.""" - from tls_fingerprint import TLSScraper - - s = TLSScraper() - if not s.is_available(): - return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"} - result = await s.fetch(url, impersonate=impersonate, proxy=proxy) - return result -@app.post( - "/v1/tls/rotate", - tags=["Advanced"], - summary="Try multiple browser fingerprints until one succeeds", -) -async def tls_rotate( - url: str = Body(...), - proxy: str = Body(""), -) -> dict[str, Any]: - """Try multiple browser fingerprints until one succeeds (anti-fingerprint rotation).""" - from tls_fingerprint import TLSScraper - - s = TLSScraper() - if not s.is_available(): - return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"} - result = await s.fetch_with_rotation(url, proxy=proxy) - return result -@app.post( - "/v1/camoufox/fetch", tags=["Advanced"], summary="Fetch with Camoufox anti-detection Firefox" -) -async def camoufox_fetch( - url: str = Body(...), - profile: str = Body("chrome_windows"), - wait_selector: str = Body(""), - proxy: str = Body(""), -) -> dict[str, Any]: - """Fetch a URL using Camoufox (Firefox anti-detection browser). - - Camoufox patches Firefox at the source level for maximum stealth. - This is more effective than Playwright for sites with advanced - fingerprinting (DataDome, PerimeterX, advanced Cloudflare). - """ - from camoufox_integration import CamoufoxBrowser - - b = CamoufoxBrowser() - if not b.is_available(): - return { - "success": False, - "error": "camoufox not installed. Run: pip install camoufox && python -m camoufox fetch", - } - result = await b.fetch(url, profile=profile, wait_selector=wait_selector, proxy=proxy) - return result -@app.post( - "/v1/graphql/discover", tags=["Advanced"], summary="Discover GraphQL endpoints for a site" -) -async def graphql_discover(base_url: str = Body(...)) -> dict[str, Any]: - """Auto-discover GraphQL endpoints for a website.""" - from graphql_discovery import GraphQLDiscovery - - g = GraphQLDiscovery() - found = await g.discover(base_url) - return {"success": True, "data": {"found": found, "count": len(found)}} -@app.post( - "/v1/graphql/introspect", tags=["Advanced"], summary="Run GraphQL introspection on an endpoint" -) -async def graphql_introspect(endpoint: str = Body(...)) -> dict[str, Any]: - """Run GraphQL introspection query against a discovered endpoint.""" - from graphql_discovery import GraphQLDiscovery - - g = GraphQLDiscovery() - result = await g.introspect(endpoint) - return {"success": "error" not in result, "data": result} -@app.post("/v1/graphql/query", tags=["Advanced"], summary="Execute a GraphQL query") -async def graphql_query( - endpoint: str = Body(...), - query: str = Body(...), - variables: dict[str, Any] | None = Body(None), -) -> dict[str, Any]: - """Execute a GraphQL query against a discovered endpoint.""" - from graphql_discovery import GraphQLDiscovery - - g = GraphQLDiscovery() - result = await g.query(endpoint, query, variables) - return {"success": "error" not in result, "data": result} -@app.post( - "/v1/schema/extract", - tags=["Advanced"], - summary="Extract Schema.org structured data from a page", -) -async def schema_extract(url: str = Body(...)) -> dict[str, Any]: - """Extract Schema.org/JSON-LD/Microdata/RDFa structured data from a URL. - Most modern sites embed structured data โ€” extract it directly instead of - scraping HTML. 100x faster and more accurate.""" - from client import get_client - from schema_extraction import SchemaExtractor - - client = await get_client() - resp = await client.get(url, timeout=30) - e = SchemaExtractor() - result = e.extract_all(resp.text) - return {"success": True, "data": result} -@app.post( - "/v1/schema/extract-html", - tags=["Advanced"], - summary="Extract Schema.org structured data from raw HTML", -) -async def schema_extract_html(html: str = Body(...)) -> dict[str, Any]: - """Extract Schema.org/JSON-LD/Microdata/RDFa from a raw HTML string. - Useful when you've already fetched the page and want to extract structured data.""" - from schema_extraction import SchemaExtractor - - e = SchemaExtractor() - result = e.extract_all(html) - return {"success": True, "data": result} -@app.post( - "/v1/ws/scrape", - tags=["Advanced"], - summary="Scrape data from a WebSocket endpoint", -) -async def ws_scrape( - url: str = Body(...), - max_messages: int = Body(100), - timeout: int = Body(30), - message_filter: str = Body(""), -) -> dict[str, Any]: - """Connect to a WebSocket and capture the data stream. - Useful for SPAs and real-time apps that load data via WebSocket.""" - from websocket_scraper import WebSocketScraper - - s = WebSocketScraper(max_messages=max_messages, timeout=timeout) - result = await s.scrape_websocket(url, message_filter=message_filter) - return {"success": result.get("success", False), "data": result} -@app.post( - "/v1/sse/scrape", - tags=["Advanced"], - summary="Scrape data from a Server-Sent Events endpoint", -) -async def sse_scrape( - url: str = Body(...), - max_events: int = Body(50), - timeout: int = Body(30), - event_filter: str = Body(""), -) -> dict[str, Any]: - """Connect to a Server-Sent Events endpoint and capture events.""" - from websocket_scraper import WebSocketScraper - - s = WebSocketScraper(timeout=timeout) - result = await s.scrape_sse(url, event_filter=event_filter, max_events=max_events) - return {"success": result.get("success", False), "data": result} # โ”€โ”€ Advanced: Cookie Warming, PDF, OCR, Dedup, Behavior โ”€โ”€ -@app.post( - "/v1/cookies/warm", - tags=["Advanced"], - summary="Warm cookies for a domain by browsing legitimate pages", -) -async def warm_cookies( - target_domain: str = Body(...), - pages_to_visit: int = Body(3), -) -> dict[str, Any]: - """Pre-age cookies for a domain to bypass anti-bot detection. - Visits legitimate pages first to build realistic browsing history.""" - from cookie_warmer import CookieWarmer - - w = CookieWarmer() - result = await w.warm_for_site(target_domain, pages_to_visit=pages_to_visit) - return {"success": result.get("success", False), "data": result} -@app.get( - "/v1/cookies/sessions", - tags=["Advanced"], - summary="List all warmed cookie sessions", -) -async def list_cookie_sessions() -> dict[str, Any]: - from cookie_warmer import CookieWarmer - - w = CookieWarmer() - return {"success": True, "data": w.list_sessions()} -@app.post( - "/v1/pdf/extract", - tags=["Advanced"], - summary="Extract tables and text from a PDF", -) -async def extract_pdf( - pdf_url: str = Body(...), - method: str = Body("pdfplumber"), -) -> dict[str, Any]: - """Download a PDF and extract structured tables and text.""" - from client import get_client - from pdf_extractor import PDFTableExtractor - - client = await get_client() - try: - resp = await client.get(pdf_url, timeout=60) - if not resp.is_success: - return { - "success": False, - "error": f"Failed to download: HTTP {resp.status_code}", - } - e = PDFTableExtractor() - result = e.extract(resp.content, method=method) - return {"success": "error" not in result, "data": result} - except (httpx.HTTPError, httpx.RequestError) as e: - return {"success": False, "error": str(e)[:300]} -@app.post( - "/v1/ocr/extract", - tags=["Advanced"], - summary="Extract text from an image using Tesseract", -) -async def ocr_extract( - image_url: str = Body(""), - image_base64: str = Body(""), -) -> dict[str, Any]: - """Extract text from an image via URL or base64.""" - from ocr_extractor import ImageOCR - - o = ImageOCR() - if image_url: - result = await o.extract_from_url(image_url) - elif image_base64: - data = base64.b64decode(image_base64.split(",", 1)[-1]) - result = o.extract_from_bytes(data) - else: - return {"success": False, "error": "Provide image_url or image_base64"} - return result -@app.post( - "/v1/dedup/check", - tags=["Advanced"], - summary="Check if content is a near-duplicate using SimHash", -) -async def check_duplicate( - text: str = Body(...), - threshold: float = Body(0.85), -) -> dict[str, Any]: - """Check if text is a near-duplicate of recently seen content using SimHash.""" - from dedup import SimHash - - h = SimHash.hash(text) - return { - "success": True, - "data": { - "hash": h, - "text_length": len(text), - "threshold": threshold, - }, - } -@app.get( - "/v1/behavior/simulate", - tags=["Advanced"], - summary="Generate human-like behavior patterns for testing", -) -async def get_behavior_simulation(action: str = "mouse") -> dict[str, Any]: - """Generate realistic human behavior patterns (mouse path, scroll, typing, etc.)""" - from behavioral_biometrics import behavior - - if action == "mouse": - path = behavior.mouse_path((0, 0), (800, 600)) - elif action == "scroll": - path = behavior.scroll_pattern(3000) - elif action == "typing": - path = behavior.typing_pattern("the quick brown fox jumps over") - elif action == "click": - return { - "success": True, - "data": {"delay_ms": behavior.click_decision_delay() * 1000}, - } - else: - return { - "success": False, - "error": f"Unknown action: {action}. Use mouse|scroll|typing|click", - } - return {"success": True, "data": path} # โ”€โ”€ x402 payment processing โ”€โ”€ -@app.post( - "/v1/x402/pay", - tags=["x402"], - summary="Process x402 payment and get access token", -) -async def x402_pay( - operation: str = Body(...), - tx_hash: str = Body(...), - payer_wallet: str = Body(...), - network: str = Body(""), - asset: str = Body(""), - amount_usd: float = Body(0.0), -) -> dict[str, Any]: - """Process an x402 payment and return an access token. - - Flow: - 1. User gets 402 from a paid endpoint - 2. User sends USDC to the wallet in the 402 response - 3. User calls this endpoint with the tx_hash - 4. Pry verifies the transaction through the facilitator router - 5. Returns access token (payment_id) to use in X-Payment-Hash header - """ - from x402 import X402Handler - - h = X402Handler() - payment_id = uuid.uuid4().hex[:12] - verify_result = await h.verify_payment( - payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd - ) - settlement = None - if verify_result.get("verified"): - settlement = await h.settle_payment( - payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd - ) - return { - "success": verify_result.get("verified", False), - "data": { - "payment_id": payment_id, - "tx_hash": tx_hash, - "verified": verify_result, - "settlement": settlement, - "payer_wallet": payer_wallet, - "use_in_header": {"X-Payment-ID": payment_id, "X-Payment-Hash": tx_hash}, - }, - } # โ”€โ”€ Actor marketplace โ”€โ”€ -@app.post( - "/v1/actors/create", - tags=["Marketplace"], - summary="Create a new actor", -) -async def create_actor( - name: str = Body(...), - description: str = Body(...), - template_id: str = Body(""), - code: str = Body(""), - price_per_run: float = Body(0.0), - visibility: str = Body("private"), - schedule_cron: str = Body(""), - tags: list[str] | None = Body(None), -) -> dict[str, Any]: - """Create a new actor in the marketplace.""" - from actor_marketplace import ActorMarketplace, ActorVisibility - - m = ActorMarketplace() - actor = m.create( - name, - description, - template_id, - code, - price_per_run, - ActorVisibility(visibility), - schedule_cron, - tags or [], - ) - return {"success": True, "data": actor.to_dict()} -@app.get( - "/v1/actors", - tags=["Marketplace"], - summary="List actors in the marketplace", -) -async def list_actors(visibility: str = "", tag: str = "") -> dict[str, Any]: - """List actors. Filter by visibility (public/private) and tag.""" - from actor_marketplace import ActorMarketplace - - m = ActorMarketplace() - return {"success": True, "data": m.list(visibility, tag)} -@app.post( - "/v1/actors/{actor_id}/run", - tags=["Marketplace"], - summary="Run an actor", -) -async def run_actor(actor_id: str, inputs: dict[str, Any] | None = Body(None)) -> dict[str, Any]: - """Run an actor with the provided inputs.""" - from actor_marketplace import ActorMarketplace - - m = ActorMarketplace() - return await m.run(actor_id, inputs or {}) # โ”€โ”€ Webhook delivery โ”€โ”€ -@app.post( - "/v1/webhooks/test", - tags=["Webhooks"], - summary="Test webhook delivery", -) -async def test_webhook( - url: str = Body(...), - payload: dict[str, Any] | None = Body(None), -) -> dict[str, Any]: - """Test webhook delivery to a URL with HMAC signing.""" - from webhook_delivery import WebhookDelivery - - w = WebhookDelivery() - return await w.deliver(url, payload or {"test": True, "timestamp": "now"}, "test.event") -@app.get( - "/v1/webhooks/dead-letter", - tags=["Webhooks"], - summary="Get failed webhook deliveries", -) -async def get_dead_letter() -> dict[str, Any]: - """Get the dead letter queue of failed webhook deliveries.""" - from webhook_delivery import WebhookDelivery - - w = WebhookDelivery() - return {"success": True, "data": w.retry_dead_letter()} if __name__ == "__main__": diff --git a/openapi.json b/openapi.json index 0ec45cc..83291b3 100644 --- a/openapi.json +++ b/openapi.json @@ -6,4454 +6,6 @@ "version": "3.0.0" }, "paths": { - "/health": { - "get": { - "tags": [ - "Health" - ], - "summary": "Full health check with dependency status", - "description": "Comprehensive health check \u2014 probes Ollama, FlareSolverr, Redis.", - "operationId": "health_check_health_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, - "/live": { - "get": { - "tags": [ - "Health" - ], - "summary": "Kubernetes liveness probe", - "description": "Simple liveness \u2014 always returns 200 if the process is running.", - "operationId": "live_live_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": { - "type": "string" - }, - "type": "object", - "title": "Response Live Live Get" - } - } - } - } - } - } - }, - "/ready": { - "get": { - "tags": [ - "Health" - ], - "summary": "Kubernetes readiness probe", - "description": "Readiness \u2014 checks critical dependencies.", - "operationId": "ready_ready_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, - "/v0/stats": { - "get": { - "tags": [ - "Stats" - ], - "summary": "Get cache, rate limiter, and session stats", - "operationId": "stats_v0_stats_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Stats V0 Stats Get" - } - } - } - } - } - } - }, - "/v1/costing/dashboard": { - "get": { - "tags": [ - "Costing" - ], - "summary": "Get cost analytics dashboard", - "description": "Get the full cost analytics dashboard.\n\nIncludes:\n- Monthly spend breakdown by operation type\n- Projected end-of-month cost\n- Daily spend for last 7 days\n- Cache efficiency metrics\n- Smart schedule recommendations", - "operationId": "cost_dashboard_v1_costing_dashboard_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Cost Dashboard V1 Costing Dashboard Get" - } - } - } - } - } - } - }, - "/v1/costing/usage": { - "get": { - "tags": [ - "Costing" - ], - "summary": "Get usage breakdown", - "description": "Get detailed usage breakdown for a specific month.", - "operationId": "cost_usage_v1_costing_usage_get", - "parameters": [ - { - "name": "year", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Year" - } - }, - { - "name": "month", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Month" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Cost Usage V1 Costing Usage Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/costing/record": { - "post": { - "tags": [ - "Costing" - ], - "summary": "Record a usage event", - "description": "Record a usage event for cost tracking.\n\nOperations: scrape_direct, scrape_flaresolverr, scrape_playwright,\ncrawl_page, llm_call, vision_call, extraction_css, bandwidth_mb", - "operationId": "record_usage_endpoint_v1_costing_record_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_record_usage_endpoint_v1_costing_record_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Record Usage Endpoint V1 Costing Record Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/costing/costs": { - "post": { - "tags": [ - "Costing" - ], - "summary": "Update per-operation cost table", - "description": "Update the per-operation cost table with custom prices.", - "operationId": "update_costs_v1_costing_costs_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": { - "type": "number" - }, - "type": "object", - "title": "Costs" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Update Costs V1 Costing Costs Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/freshness/check": { - "post": { - "tags": [ - "Freshness" - ], - "summary": "Check if content has changed since last scrape", - "description": "Check if content has changed since the last scrape.\n\nUses content fingerprinting (SHA256) to detect changes.\nIf no content is provided, does a quick HEAD check instead.", - "operationId": "freshness_check_v1_freshness_check_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_freshness_check_v1_freshness_check_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Freshness Check V1 Freshness Check Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/freshness/frequency": { - "post": { - "tags": [ - "Freshness" - ], - "summary": "Get adaptive scrape frequency recommendation", - "description": "Get an adaptive scrape frequency recommendation based on content volatility.\n\nVolatile pages (frequent changes) get shorter intervals.\nStable pages get longer intervals to save costs.", - "operationId": "freshness_frequency_v1_freshness_frequency_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_freshness_frequency_v1_freshness_frequency_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Freshness Frequency V1 Freshness Frequency Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/freshness/dashboard": { - "get": { - "tags": [ - "Freshness" - ], - "summary": "Get content staleness dashboard", - "description": "Get the content staleness dashboard.\n\nShows all tracked URLs, their last check time, age, and whether\nthey're stale (not checked in 24h).", - "operationId": "freshness_dashboard_v1_freshness_dashboard_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Freshness Dashboard V1 Freshness Dashboard Get" - } - } - } - } - } - } - }, - "/v1/scrape": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Scrape a single URL", - "description": "Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.", - "operationId": "scrape_v1_scrape_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ScrapeRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Scrape V1 Scrape Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/detect-block": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Detect if a site is blocking the scraper", - "description": "Detect what kind of anti-bot protection a site is using.\n\nReturns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.\nUseful for debugging scraping issues.", - "operationId": "detect_block_v1_detect_block_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Detect Block V1 Detect Block Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/ultimate-scrape": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Scrape with 10-tier anti-bot fallback system", - "description": "Scrape any URL using Pry's ultimate 10-tier anti-detection system.\n\nAutomatically tries: direct \u2192 cloudscraper \u2192 FlareSolverr \u2192\nundetected-chromedriver \u2192 Playwright \u2192 Googlebot \u2192 Archive.org \u2192 Google Cache\n\nReturns the first successful result with the method used.", - "operationId": "ultimate_scrape_v1_ultimate_scrape_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_ultimate_scrape_v1_ultimate_scrape_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Ultimate Scrape V1 Ultimate Scrape Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/capture/network": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Extract API calls and network patterns from a page", - "description": "Extract API calls, GraphQL queries, and network patterns from a page.\n\nUseful for understanding how SPAs load data and finding hidden API endpoints.", - "operationId": "capture_network_v1_capture_network_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Capture Network V1 Capture Network Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/capture/lazy": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Detect and handle lazy-loaded content", - "description": "Detect lazy loading and infinite scroll patterns on a page.\n\nOptionally generate JS to auto-scroll and load all content.", - "operationId": "detect_lazy_content_v1_capture_lazy_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_detect_lazy_content_v1_capture_lazy_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Detect Lazy Content V1 Capture Lazy Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/crawl": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Crawl multiple pages from a URL", - "description": "Crawl multiple pages from a URL. Supports async webhooks.", - "operationId": "crawl_v1_crawl_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CrawlRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Crawl V1 Crawl Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/crawl/adaptive": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Crawl with adaptive stopping based on content relevance", - "description": "Crawl a website with adaptive stopping.\n\nUses information foraging theory to decide when to stop:\n- Stops when content relevance drops below threshold\n- Stops when information gain diminishes\n- Respects max_pages and max_depth limits\n- Ideal for targeted data collection (pricing, docs, products)", - "operationId": "adaptive_crawl_v1_crawl_adaptive_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_adaptive_crawl_v1_crawl_adaptive_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Adaptive Crawl V1 Crawl Adaptive Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/map": { - "post": { - "tags": [ - "Scraping" - ], - "summary": "Discover URLs on a site", - "description": "Discover URLs on a site.", - "operationId": "map_pages_v1_map_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/MapRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Map Pages V1 Map Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/parse": { - "post": { - "tags": [ - "Parsing" - ], - "summary": "Parse a document (PDF, DOCX, image, CSV, JSON)", - "description": "Parse a document (PDF, DOCX, image, CSV, JSON) to text.", - "operationId": "parse_document_v1_parse_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ParseRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Parse Document V1 Parse Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/markdown": { - "post": { - "tags": [ - "Parsing" - ], - "summary": "Generate markdown with content filtering strategies", - "description": "Generate markdown with configurable content filtering.\n\nModes:\n- raw: Unfiltered markdown\n- fit: Prune boilerplate (nav, ads, footers)\n- bm25: Filter by BM25 relevance to query (requires query param)", - "operationId": "generate_markdown_v1_markdown_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_generate_markdown_v1_markdown_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Generate Markdown V1 Markdown Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/automate": { - "post": { - "tags": [ - "Automation" - ], - "summary": "Execute browser automation steps", - "description": "Execute browser automation steps. Sessions persist for login flows.", - "operationId": "automate_v1_automate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AutomateRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Automate V1 Automate Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/screenshot": { - "post": { - "tags": [ - "Automation" - ], - "summary": "Take a screenshot of a URL", - "description": "Take a screenshot of a URL. Returns base64 PNG.", - "operationId": "screenshot_v1_screenshot_post", - "parameters": [ - { - "name": "session_id", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Session Id" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_screenshot_v1_screenshot_post" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Screenshot V1 Screenshot Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/vision": { - "post": { - "tags": [ - "Vision" - ], - "summary": "Analyze an image with a free vision model", - "description": "Analyze an image with a free vision model.\n\nProvide ONE of: image (base64 or data URI), url (auto-screenshot),\nor file_path (local PNG/JPG).\n\nAuto-falls-back across 5 free OpenRouter vision models if the\nrequested one is rate-limited.", - "operationId": "vision_v1_vision_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_vision_v1_vision_post" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Vision V1 Vision Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/vision/models": { - "get": { - "tags": [ - "Vision" - ], - "summary": "List free vision-capable models on OpenRouter", - "description": "List free vision-capable models on OpenRouter.", - "operationId": "vision_models_v1_vision_models_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Vision Models V1 Vision Models Get" - } - } - } - } - } - } - }, - "/v1/session/create": { - "post": { - "tags": [ - "Sessions" - ], - "summary": "Create a persistent browser session", - "description": "Create a persistent browser session.", - "operationId": "create_session_v1_session_create_post", - "parameters": [ - { - "name": "persist", - "in": "query", - "required": false, - "schema": { - "type": "boolean", - "default": true, - "title": "Persist" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Create Session V1 Session Create Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/session/destroy": { - "post": { - "tags": [ - "Sessions" - ], - "summary": "Destroy a browser session with optional save", - "description": "Destroy a browser session. Optionally save state first.", - "operationId": "destroy_session_v1_session_destroy_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_destroy_session_v1_session_destroy_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Destroy Session V1 Session Destroy Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/sessions": { - "get": { - "tags": [ - "Sessions" - ], - "summary": "List all persistent sessions", - "description": "List all persistent sessions (active + saved).", - "operationId": "list_sessions_v1_sessions_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Sessions V1 Sessions Get" - } - } - } - } - } - } - }, - "/v1/session/save": { - "post": { - "tags": [ - "Sessions" - ], - "summary": "Save current session state to disk", - "description": "Save a session's current state (cookies, storage) to disk.", - "operationId": "save_session_state_v1_session_save_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Session Id" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Save Session State V1 Session Save Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/session/restore": { - "post": { - "tags": [ - "Sessions" - ], - "summary": "Restore a saved session", - "description": "Restore a session from disk into a browser context.", - "operationId": "restore_session_v1_session_restore_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Session Id" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Restore Session V1 Session Restore Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/batch": { - "post": { - "tags": [ - "Batch" - ], - "summary": "Scrape multiple URLs in parallel", - "description": "Scrape multiple URLs in parallel. Firecrawl charges extra for batch.", - "operationId": "batch_scrape_v1_batch_post", - "parameters": [ - { - "name": "timeout", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 30, - "title": "Timeout" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string" - }, - "title": "Urls" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Batch Scrape V1 Batch Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/compare": { - "post": { - "tags": [ - "Analysis" - ], - "summary": "Compare content of two URLs", - "description": "Scrape two URLs and compare their content. Shows additions, deletions, changes.", - "operationId": "compare_v1_compare_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_compare_v1_compare_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Compare V1 Compare Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/watch": { - "post": { - "tags": [ - "Analysis" - ], - "summary": "Watch a page for changes", - "description": "Watch a page for changes. Accepts a webhook URL for notification.\nFirst call registers the watch, subsequent calls compare and notify.\nFirecrawl charges $99/mo for this feature.", - "operationId": "watch_page_v1_watch_post", - "parameters": [ - { - "name": "interval", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 3600, - "title": "Interval" - } - }, - { - "name": "selector", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "", - "title": "Selector" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_watch_page_v1_watch_post" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Watch Page V1 Watch Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/export": { - "post": { - "tags": [ - "Export" - ], - "summary": "Export scraped content in multiple formats", - "description": "Export scraped content in multiple formats: json, csv, txt, rss.\nFirecrawl only does markdown.", - "operationId": "export_data_v1_export_post", - "parameters": [ - { - "name": "format", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "json", - "title": "Format" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Export Data V1 Export Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/shadow-dom": { - "post": { - "tags": [ - "Parsing" - ], - "summary": "Extract content from Shadow DOM", - "description": "Scrape a page and extract content from Shadow DOM components.\n\nUseful for modern web apps built with Lit, web components, or\nframeworks that use Shadow DOM encapsulation.", - "operationId": "extract_shadow_dom_v1_shadow_dom_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_extract_shadow_dom_v1_shadow_dom_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Extract Shadow Dom V1 Shadow Dom Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/extract-table": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Extract HTML tables as structured data", - "description": "Extract HTML tables from a page as structured data.\nFirecrawl doesn't support table extraction at all.", - "operationId": "extract_table_v1_extract_table_post", - "parameters": [ - { - "name": "table_index", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 0, - "title": "Table Index" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Extract Table V1 Extract Table Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/links": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Analyze links on a page", - "description": "Analyze all links on a page \u2014 internal, external, broken, social.\nFirecrawl only has basic map functionality.", - "operationId": "analyze_links_v1_links_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Analyze Links V1 Links Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/seo": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "SEO analysis of a page", - "description": "SEO analysis of a page: title, description, headings, images, keywords, readability.\nFirecrawl has zero SEO features.", - "operationId": "analyze_seo_v1_seo_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Analyze Seo V1 Seo Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/summarize": { - "post": { - "tags": [ - "Analysis" - ], - "summary": "AI summarize scraped content", - "description": "Scrape + AI summarize using local Ollama. Free, private.", - "operationId": "summarize_v1_summarize_post", - "parameters": [ - { - "name": "max_words", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 100, - "title": "Max Words" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Summarize V1 Summarize Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/diff": { - "post": { - "tags": [ - "Analysis" - ], - "summary": "Track page changes over time", - "description": "Track page changes over time. Returns diff from last scrape.", - "operationId": "diff_v1_diff_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_diff_v1_diff_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Diff V1 Diff Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/schema": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Extract Schema.org/JSON-LD structured data", - "description": "Extract Schema.org/JSON-LD structured data from a page.", - "operationId": "extract_schema_v1_schema_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Extract Schema V1 Schema Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/emails": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Find email addresses on a page", - "description": "Find all email addresses on a page.", - "operationId": "find_emails_v1_emails_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Find Emails V1 Emails Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/email/scrape": { - "post": { - "tags": [ - "Email" - ], - "summary": "Extract data from an email (subject + body)", - "description": "Extract structured data from an email subject and body.\n\nAuto-classifies as: order_confirmation, invoice, receipt,\nshipping_notification, subscription, or other.\n\nExtracts: order numbers, amounts, tracking numbers, dates, addresses.", - "operationId": "scrape_email_v1_email_scrape_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_scrape_email_v1_email_scrape_post" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Scrape Email V1 Email Scrape Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/email/gmail": { - "post": { - "tags": [ - "Email" - ], - "summary": "Fetch and extract data from Gmail inbox", - "description": "Connect to Gmail and extract structured data from emails.\n\nRequires a Gmail OAuth2 access token with scope:\nhttps://www.googleapis.com/auth/gmail.readonly\n\nExtracts order confirmations, invoices, receipts, shipping notifications.", - "operationId": "fetch_gmail_v1_email_gmail_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_fetch_gmail_v1_email_gmail_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Fetch Gmail V1 Email Gmail Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/email/outlook": { - "post": { - "tags": [ - "Email" - ], - "summary": "Fetch and extract data from Outlook inbox", - "description": "Connect to Outlook/Office 365 and extract structured data from emails.\n\nRequires a Microsoft Graph OAuth2 access token with scope:\nhttps://graph.microsoft.com/Mail.Read\n\nExtracts order confirmations, invoices, receipts, shipping notifications.", - "operationId": "fetch_outlook_v1_email_outlook_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_fetch_outlook_v1_email_outlook_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Fetch Outlook V1 Email Outlook Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/categorize": { - "post": { - "tags": [ - "Analysis" - ], - "summary": "AI-categorize scraped content", - "description": "AI-categorize scraped content into topic tags.", - "operationId": "categorize_v1_categorize_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Categorize V1 Categorize Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/destinations": { - "get": { - "tags": [ - "Integrations" - ], - "summary": "List supported data destinations", - "description": "List all supported data destinations and their config requirements.", - "operationId": "list_destinations_v1_destinations_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Destinations V1 Destinations Get" - } - } - } - } - } - } - }, - "/v1/destination/send": { - "post": { - "tags": [ - "Integrations" - ], - "summary": "Send scraped data to a business destination", - "description": "Send extracted data to a business destination in one click.\n\nDestinations:\n- slack: Send to Slack channel via webhook\n- googlesheets: Write to Google Sheets (requires credentials)\n- airtable: Write to Airtable base (requires API key)\n- email: Send via email (requires SMTP config)\n\nConfig varies by destination:\n- slack: {\"webhook_url\": \"https://hooks.slack.com/...\"}\n- googlesheets: {\"spreadsheet_id\": \"...\", \"credentials_json\": \"...\"}\n- airtable: {\"base_id\": \"...\", \"table_name\": \"Table 1\", \"api_key\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"subject\": \"Data Export\"}", - "operationId": "send_to_destination_v1_destination_send_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_send_to_destination_v1_destination_send_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Send To Destination V1 Destination Send Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/scrape-and-send": { - "post": { - "tags": [ - "Integrations" - ], - "summary": "Scrape a URL and send to a destination", - "description": "Scrape a URL and send the results to a business destination in one step.\n\nCombines /v1/scrape + /v1/destination/send into a single call.\nPerfect for non-technical users who just want data in their tools.", - "operationId": "scrape_and_send_v1_scrape_and_send_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_scrape_and_send_v1_scrape_and_send_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Scrape And Send V1 Scrape And Send Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/alert/send": { - "post": { - "tags": [ - "Alerts" - ], - "summary": "Send an alert to any channel", - "description": "Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email.\n\nConfig varies by channel:\n- slack: {\"webhook_url\": \"...\"}\n- discord: {\"webhook_url\": \"...\"}\n- teams: {\"webhook_url\": \"...\"}\n- telegram: {\"bot_token\": \"...\", \"chat_id\": \"...\"}\n- sms: {\"phone\": \"+1234567890\", \"twilio_sid\": \"...\", \"twilio_token\": \"...\", \"twilio_from\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"smtp_host\": \"...\", \"smtp_user\": \"...\"}", - "operationId": "send_alert_endpoint_v1_alert_send_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_send_alert_endpoint_v1_alert_send_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Send Alert Endpoint V1 Alert Send Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/alert/channels": { - "get": { - "tags": [ - "Alerts" - ], - "summary": "List supported alert channels", - "description": "List all supported alert channels and their config requirements.", - "operationId": "list_channels_v1_alert_channels_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Channels V1 Alert Channels Get" - } - } - } - } - } - } - }, - "/v1/job/{job_id}": { - "get": { - "tags": [ - "Jobs" - ], - "summary": "Get async job status and result", - "description": "Get async job status and result.", - "operationId": "get_job_v1_job__job_id__get", - "parameters": [ - { - "name": "job_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Job Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Job V1 Job Job Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/config": { - "get": { - "tags": [ - "Config" - ], - "summary": "Get current Pry configuration", - "description": "Get current Pry configuration.", - "operationId": "get_config_v1_config_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Get Config V1 Config Get" - } - } - } - } - } - }, - "post": { - "tags": [ - "Config" - ], - "summary": "Update Pry configuration at runtime", - "description": "Update Pry configuration at runtime.", - "operationId": "update_config_v1_config_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Updates" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Update Config V1 Config Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/config/profile/tor": { - "post": { - "tags": [ - "Config" - ], - "summary": "Enable Tor routing for all requests", - "description": "Enable Tor routing for all requests.", - "operationId": "enable_tor_v1_config_profile_tor_post", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Enable Tor V1 Config Profile Tor Post" - } - } - } - } - } - } - }, - "/v1/batch-file": { - "post": { - "tags": [ - "Batch" - ], - "summary": "Batch scrape URLs from a file with templates", - "operationId": "batch_from_file_v1_batch_file_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchFileRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Batch From File V1 Batch File Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/record/start": { - "post": { - "tags": [ - "Recorder" - ], - "summary": "Start recording browser actions", - "operationId": "start_recording_v1_record_start_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Session Id" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Start Recording V1 Record Start Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/record/step": { - "post": { - "tags": [ - "Recorder" - ], - "summary": "Record a browser action step", - "operationId": "record_step_v1_record_step_post", - "parameters": [ - { - "name": "selector", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "", - "title": "Selector" - } - }, - { - "name": "value", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "", - "title": "Value" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_record_step_v1_record_step_post" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Record Step V1 Record Step Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/record/export": { - "post": { - "tags": [ - "Recorder" - ], - "summary": "Export recording as script", - "operationId": "export_recording_v1_record_export_post", - "parameters": [ - { - "name": "fmt", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "json", - "title": "Fmt" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Session Id" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Export Recording V1 Record Export Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/record/clear": { - "post": { - "tags": [ - "Recorder" - ], - "summary": "Clear recorded actions", - "operationId": "clear_recording_v1_record_clear_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Session Id" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Clear Recording V1 Record Clear Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/transform": { - "post": { - "tags": [ - "Transform" - ], - "summary": "Transform data to multiple formats", - "operationId": "transform_data_v1_transform_post", - "parameters": [ - { - "name": "format", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "csv", - "title": "Format" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": true - }, - "title": "Data" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Transform Data V1 Transform Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/run": { - "post": { - "tags": [ - "Execute" - ], - "summary": "Execute a Pryfile", - "operationId": "run_pryfile_v1_run_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Path", - "default": "pry.yml" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Run Pryfile V1 Run Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/jobs": { - "get": { - "tags": [ - "Jobs" - ], - "summary": "List jobs in a Pryfile", - "operationId": "list_jobs_v1_jobs_get", - "parameters": [ - { - "name": "path", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "pry.yml", - "title": "Path" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response List Jobs V1 Jobs Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/dashboard": { - "get": { - "tags": [ - "Dashboard" - ], - "summary": "Pry health and performance dashboard", - "operationId": "dashboard_dashboard_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - } - } - } - }, - "/v1/suggest": { - "post": { - "tags": [ - "Analysis" - ], - "summary": "AI-suggest schema fields for a URL", - "operationId": "suggest_schema_v1_suggest_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Data" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Suggest Schema V1 Suggest Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/breaker/status": { - "post": { - "tags": [ - "Circuit Breaker" - ], - "summary": "Get circuit breaker status", - "operationId": "breaker_status_v1_breaker_status_post", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Breaker Status V1 Breaker Status Post" - } - } - } - } - } - } - }, - "/v1/breaker/reset": { - "post": { - "tags": [ - "Circuit Breaker" - ], - "summary": "Reset circuit breaker for a domain", - "operationId": "breaker_reset_v1_breaker_reset_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Domain", - "default": "" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Breaker Reset V1 Breaker Reset Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/extract": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Extract structured fields from a URL", - "operationId": "extract_stable_v1_extract_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Data" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Extract Stable V1 Extract Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/extract/css": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Extract structured data with CSS selectors (no LLM)", - "description": "Extract structured JSON from a URL using CSS selector schema.\n\nSchema format:\n{\n \"name\": \"products\",\n \"base_selector\": \".product-card\",\n \"fields\": [\n {\"name\": \"title\", \"selector\": \"h3\", \"type\": \"text\"},\n {\"name\": \"price\", \"selector\": \".price\", \"type\": \"text\", \"transform\": \"float\"},\n {\"name\": \"link\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"},\n {\"name\": \"in_stock\", \"selector\": \".stock\", \"type\": \"exists\"},\n ]\n}", - "operationId": "extract_css_v1_extract_css_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_extract_css_v1_extract_css_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Extract Css V1 Extract Css Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/extract/llm": { - "post": { - "tags": [ - "Extraction" - ], - "summary": "Extract with LLM + chunking strategies", - "description": "Extract structured data using LLM with intelligent chunking.\n\nChunks content by strategy (topic/sentence/regex), optionally filters\nby relevance to query, then extracts from each chunk.", - "operationId": "extract_llm_v1_extract_llm_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_extract_llm_v1_extract_llm_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Extract Llm V1 Extract Llm Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/pipeline/hook": { - "post": { - "tags": [ - "Pipeline" - ], - "summary": "Register a hook function", - "description": "Register a JavaScript hook function at a pipeline hook point.\n\nHook points: before_scrape, after_response, before_parse, after_parse,\n before_extract, after_extract, before_return, on_error", - "operationId": "register_hook_v1_pipeline_hook_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_register_hook_v1_pipeline_hook_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Register Hook V1 Pipeline Hook Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/pipeline/hooks": { - "get": { - "tags": [ - "Pipeline" - ], - "summary": "List all registered hooks", - "description": "List all registered hooks in the pipeline.", - "operationId": "list_pipeline_hooks_v1_pipeline_hooks_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Pipeline Hooks V1 Pipeline Hooks Get" - } - } - } - } - } - } - }, - "/v1/pipeline/run": { - "post": { - "tags": [ - "Pipeline" - ], - "summary": "Run pipeline hooks for testing", - "description": "Run hooks at a given pipeline point for testing.", - "operationId": "run_pipeline_test_v1_pipeline_run_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_run_pipeline_test_v1_pipeline_run_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Run Pipeline Test V1 Pipeline Run Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/pipe": { - "post": { - "tags": [ - "Transform" - ], - "summary": "Scrape and transform via data pipeline", - "operationId": "data_pipeline_v1_pipe_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Data" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Data Pipeline V1 Pipe Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/share": { - "post": { - "tags": [ - "Share" - ], - "summary": "Share scraped content via link", - "operationId": "share_scrape_v1_share_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Data" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Share Scrape V1 Share Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/share/{share_id}": { - "get": { - "tags": [ - "Share" - ], - "summary": "View shared content", - "operationId": "view_share_share__share_id__get", - "parameters": [ - { - "name": "share_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Share Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": {} - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/compliance/check": { - "get": { - "tags": [ - "Compliance" - ], - "summary": "Get compliance check documentation", - "description": "Get information about the compliance check endpoint.", - "operationId": "compliance_docs_v1_compliance_check_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Compliance Docs V1 Compliance Check Get" - } - } - } - } - } - }, - "post": { - "tags": [ - "Compliance" - ], - "summary": "Run full compliance check on a URL", - "description": "Run a full legal compliance check on a target URL.\n\nAnalyzes:\n- robots.txt crawl permissions\n- Terms of Service classification (restrictive/permissive/moderate)\n- Jurisdiction detection (GDPR, CCPA, LGPD)\n- Sensitive data detection (PII, financial, health, contact)\n\nReturns a green/yellow/red risk score with recommendations.", - "operationId": "compliance_check_v1_compliance_check_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Compliance Check V1 Compliance Check Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/gdpr/consent": { - "post": { - "tags": [ - "GDPR" - ], - "summary": "Record user consent for data processing", - "description": "Record a user's consent for data processing (GDPR Art. 7).\n\nStores: user hash, purpose, timestamp, IP, user agent.\nConsent expires after 365 days.", - "operationId": "record_consent_endpoint_v1_gdpr_consent_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_record_consent_endpoint_v1_gdpr_consent_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Record Consent Endpoint V1 Gdpr Consent Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/gdpr/consent/{user_id}": { - "get": { - "tags": [ - "GDPR" - ], - "summary": "Check user consent status", - "description": "Check if a user has given valid consent for data processing.", - "operationId": "check_consent_endpoint_v1_gdpr_consent__user_id__get", - "parameters": [ - { - "name": "user_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "User Id" - } - }, - { - "name": "purpose", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "data_collection", - "title": "Purpose" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Check Consent Endpoint V1 Gdpr Consent User Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/gdpr/consent/revoke": { - "post": { - "tags": [ - "GDPR" - ], - "summary": "Revoke user consent", - "description": "Revoke a user's consent for a processing purpose (GDPR Art. 7(3)).", - "operationId": "revoke_consent_endpoint_v1_gdpr_consent_revoke_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_revoke_consent_endpoint_v1_gdpr_consent_revoke_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Revoke Consent Endpoint V1 Gdpr Consent Revoke Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/gdpr/deletion/request": { - "post": { - "tags": [ - "GDPR" - ], - "summary": "Request data deletion (right to erasure)", - "description": "Request deletion of all data associated with a user (GDPR Art. 17).\n\nCreates a deletion request that can be executed immediately or\nafter a holding period.", - "operationId": "request_deletion_endpoint_v1_gdpr_deletion_request_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_request_deletion_endpoint_v1_gdpr_deletion_request_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Request Deletion Endpoint V1 Gdpr Deletion Request Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/gdpr/deletion/execute": { - "post": { - "tags": [ - "GDPR" - ], - "summary": "Execute data deletion request", - "description": "Execute a pending deletion request, removing all user data.", - "operationId": "execute_deletion_endpoint_v1_gdpr_deletion_execute_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Request Id" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Execute Deletion Endpoint V1 Gdpr Deletion Execute Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/gdpr/retention": { - "get": { - "tags": [ - "GDPR" - ], - "summary": "Get data retention policy", - "description": "Get the current data retention policy.", - "operationId": "get_retention_policy_endpoint_v1_gdpr_retention_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Get Retention Policy Endpoint V1 Gdpr Retention Get" - } - } - } - } - } - } - }, - "/v1/gdpr/retention/apply": { - "post": { - "tags": [ - "GDPR" - ], - "summary": "Apply retention policy to remove expired data", - "description": "Apply the retention policy, removing all expired data.", - "operationId": "apply_retention_v1_gdpr_retention_apply_post", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Apply Retention V1 Gdpr Retention Apply Post" - } - } - } - } - } - } - }, - "/v1/gdpr/audit": { - "get": { - "tags": [ - "GDPR" - ], - "summary": "Get compliance audit log", - "description": "Get the compliance audit log for the specified period.", - "operationId": "get_audit_log_endpoint_v1_gdpr_audit_get", - "parameters": [ - { - "name": "days_back", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 7, - "title": "Days Back" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Audit Log Endpoint V1 Gdpr Audit Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/training/classify-license": { - "post": { - "tags": [ - "Training" - ], - "summary": "Classify content license for AI training", - "description": "Classify the license of scraped content for AI training compliance.\n\nReturns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use),\ntier (permissive/copyleft/restrictive/conditional), and confidence.", - "operationId": "classify_license_endpoint_v1_training_classify_license_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Text" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Classify License Endpoint V1 Training Classify License Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/training/clean": { - "post": { - "tags": [ - "Training" - ], - "summary": "Strip PII and copyright from content", - "description": "Strip PII and copyright content for AI training clean room.\n\nRemoves emails, phones, SSNs, credit cards, IPs, and (optionally) names.\nAlso strips copyright notices and near-verbatim copyright content.", - "operationId": "clean_content_v1_training_clean_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_clean_content_v1_training_clean_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Clean Content V1 Training Clean Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/training/export": { - "post": { - "tags": [ - "Training" - ], - "summary": "Export a clean AI training dataset", - "description": "Export scraped content as a clean AI training dataset.\n\nEach record should have: content, url, and optional metadata.\n\nFeatures:\n- Per-record provenance tracking (source URL, timestamp, extraction method)\n- PII stripping (email, phone, SSN, CC, IP)\n- Copyright verbatim text removal\n- License classification\n- Compliance report generation", - "operationId": "export_dataset_v1_training_export_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_export_dataset_v1_training_export_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Export Dataset V1 Training Export Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/training/compliance/{dataset_id}": { - "get": { - "tags": [ - "Training" - ], - "summary": "Generate compliance report for a dataset", - "description": "Generate a compliance report for an exported training dataset.\n\nReport includes: data provenance, PII/copyright removal stats,\nlicense classification, and legal recommendations.", - "operationId": "compliance_report_v1_training_compliance__dataset_id__get", - "parameters": [ - { - "name": "dataset_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Dataset Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Compliance Report V1 Training Compliance Dataset Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/monitor": { - "post": { - "tags": [ - "Monitoring" - ], - "summary": "Create a scheduled content monitor", - "description": "Create a scheduled monitor that tracks content changes.\n\nArgs:\n name: Human-readable monitor name\n url: Target URL to monitor\n schedule_cron: Cron expression (default: every 6 hours)\n goal: Natural language goal for meaningful-change detection\n webhook: URL to notify on meaningful changes\n use_llm: Use LLM for change judging (slower but smarter)", - "operationId": "create_monitor_endpoint_v1_monitor_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_create_monitor_endpoint_v1_monitor_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Create Monitor Endpoint V1 Monitor Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/monitor/run": { - "post": { - "tags": [ - "Monitoring" - ], - "summary": "Run a single monitor check", - "description": "Execute a monitor check immediately (outside of schedule).", - "operationId": "run_monitor_endpoint_v1_monitor_run_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Monitor Id" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Run Monitor Endpoint V1 Monitor Run Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/monitors": { - "get": { - "tags": [ - "Monitoring" - ], - "summary": "List all monitors", - "description": "List all registered monitors.", - "operationId": "list_monitors_endpoint_v1_monitors_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Monitors Endpoint V1 Monitors Get" - } - } - } - } - } - } - }, - "/v1/monitor/{monitor_id}": { - "delete": { - "tags": [ - "Monitoring" - ], - "summary": "Delete a monitor", - "description": "Delete a monitor and its snapshots.", - "operationId": "delete_monitor_endpoint_v1_monitor__monitor_id__delete", - "parameters": [ - { - "name": "monitor_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Monitor Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Delete Monitor Endpoint V1 Monitor Monitor Id Delete" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/monitor/check": { - "post": { - "tags": [ - "Monitoring" - ], - "summary": "Run all due monitors", - "description": "Check all monitors and run any that are due based on their cron schedule.", - "operationId": "run_due_monitors_v1_monitor_check_post", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Run Due Monitors V1 Monitor Check Post" - } - } - } - } - } - } - }, - "/v1/structure/check": { - "post": { - "tags": [ - "Structure" - ], - "summary": "Check if CSS selectors still match on a page", - "description": "Check if CSS selectors still match on a page.\n\nUse this to verify your scraper templates still work after\na site redesign. Returns per-selector match status.\n\nSelectors format: [{\"name\": \"title\", \"selector\": \"h1\", \"type\": \"css\"}]", - "operationId": "structure_check_v1_structure_check_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_structure_check_v1_structure_check_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Structure Check V1 Structure Check Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/structure/monitor": { - "post": { - "tags": [ - "Structure" - ], - "summary": "Monitor page structure changes over time", - "description": "Monitor page structure changes and detect broken selectors.\n\nCompares current selector match status to previous checks.\nAlerts when selectors stop matching (site redesign detected).\n\nUse this as a pre-scrape health check for your templates.", - "operationId": "structure_monitor_v1_structure_monitor_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_structure_monitor_v1_structure_monitor_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Structure Monitor V1 Structure Monitor Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/structure/check-template": { - "post": { - "tags": [ - "Structure" - ], - "summary": "Verify a scraper template still works", - "description": "Check if a pre-built scraper template still works against a URL.\n\nExtracts the template's selectors and checks each one.\nIf selectors fail, the template needs updating.", - "operationId": "structure_check_template_v1_structure_check_template_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_structure_check_template_v1_structure_check_template_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Structure Check Template V1 Structure Check Template Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/intel/snapshot": { - "post": { - "tags": [ - "Intelligence" - ], - "summary": "Record a competitor data snapshot", - "description": "Record a data snapshot for a competitor.\n\nSnapshots are stored with timestamps and used for trend analysis,\nanomaly detection, and report generation.", - "operationId": "record_intel_snapshot_v1_intel_snapshot_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_record_intel_snapshot_v1_intel_snapshot_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Record Intel Snapshot V1 Intel Snapshot Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/intel/snapshots/{competitor_id}": { - "get": { - "tags": [ - "Intelligence" - ], - "summary": "Get competitor snapshots", - "description": "Get historical snapshots for a competitor.", - "operationId": "get_intel_snapshots_v1_intel_snapshots__competitor_id__get", - "parameters": [ - { - "name": "competitor_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Competitor Id" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 50, - "title": "Limit" - } - }, - { - "name": "since_hours", - "in": "query", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Since Hours" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Intel Snapshots V1 Intel Snapshots Competitor Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/intel/analyze": { - "post": { - "tags": [ - "Intelligence" - ], - "summary": "Analyze competitor field for anomalies", - "description": "Analyze a specific field across a competitor's snapshots for anomalies.\n\nReturns statistics, z-score analysis, and anomaly detection.", - "operationId": "analyze_field_v1_intel_analyze_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_analyze_field_v1_intel_analyze_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Analyze Field V1 Intel Analyze Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/intel/report": { - "post": { - "tags": [ - "Intelligence" - ], - "summary": "Generate a competitive intelligence report", - "description": "Generate a competitive intelligence report for tracked competitors.\n\nAnalyzes changes over the specified period and returns a structured report\nwith a natural-language summary.", - "operationId": "generate_report_v1_intel_report_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_generate_report_v1_intel_report_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Generate Report V1 Intel Report Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/quality/check": { - "post": { - "tags": [ - "Quality" - ], - "summary": "Run data quality check on extraction results", - "description": "Run a full data quality check on extraction results.\n\nMetrics:\n- Completeness: what % of expected fields have values\n- Schema adherence: field types match expectations\n- Freshness: how old is the data\n- Anomaly detection: what changed since last extraction\n\nUse this to validate data BEFORE sending to downstream systems.", - "operationId": "quality_check_v1_quality_check_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_quality_check_v1_quality_check_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Quality Check V1 Quality Check Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/quality/stats": { - "get": { - "tags": [ - "Quality" - ], - "summary": "Get quality statistics for all checked URLs", - "description": "Get aggregate quality statistics across all checked URLs.", - "operationId": "quality_stats_v1_quality_stats_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Quality Stats V1 Quality Stats Get" - } - } - } - } - } - } - }, - "/v1/reconcile": { - "post": { - "tags": [ - "Reconciliation" - ], - "summary": "Reconcile records from multiple sources into unified entities", - "description": "Reconcile records from multiple sources into matched entities.\n\nMatches records across sources using identity field similarity,\nnormalizes to a unified vertical schema, and returns entity groups\nwith confidence scores.\n\nVerticals: product, job, real_estate, review", - "operationId": "reconcile_endpoint_v1_reconcile_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_reconcile_endpoint_v1_reconcile_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Reconcile Endpoint V1 Reconcile Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/reconcile/schemas": { - "get": { - "tags": [ - "Reconciliation" - ], - "summary": "List supported reconciliation schemas", - "description": "List all supported vertical schemas for entity reconciliation.", - "operationId": "list_schemas_v1_reconcile_schemas_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Schemas V1 Reconcile Schemas Get" - } - } - } - } - } - } - }, "/v1/auth/credentials": { "get": { "tags": [ @@ -4697,19 +249,85 @@ } } }, - "/v1/review/submit": { + "/health": { + "get": { + "tags": [ + "Health" + ], + "summary": "Full health check with dependency status", + "description": "Comprehensive health check \u2014 probes Ollama, FlareSolverr, Redis.", + "operationId": "health_check_health_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/live": { + "get": { + "tags": [ + "Health" + ], + "summary": "Kubernetes liveness probe", + "description": "Simple liveness \u2014 always returns 200 if the process is running.", + "operationId": "live_live_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "title": "Response Live Live Get" + } + } + } + } + } + } + }, + "/ready": { + "get": { + "tags": [ + "Health" + ], + "summary": "Kubernetes readiness probe", + "description": "Readiness \u2014 checks critical dependencies.", + "operationId": "ready_ready_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v1/scrape": { "post": { "tags": [ - "Review" + "Scraping" ], - "summary": "Submit extracted data for human review", - "description": "Submit extracted data for human review before delivery.\n\nUse this when extraction confidence is low or anomalies were detected.\nData will be held in the review queue until approved or rejected.", - "operationId": "review_submit_v1_review_submit_post", + "summary": "Scrape a single URL", + "description": "Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.", + "operationId": "scrape_v1_scrape_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Body_review_submit_v1_review_submit_post" + "$ref": "#/components/schemas/ScrapeRequest" } } }, @@ -4723,7 +341,7 @@ "schema": { "additionalProperties": true, "type": "object", - "title": "Response Review Submit V1 Review Submit Post" + "title": "Response Scrape V1 Scrape Post" } } } @@ -4741,33 +359,24 @@ } } }, - "/v1/review/{review_id}/approve": { + "/v1/detect-block": { "post": { "tags": [ - "Review" - ], - "summary": "Approve a review item", - "description": "Approve a review item, allowing data to proceed to delivery.", - "operationId": "review_approve_v1_review__review_id__approve_post", - "parameters": [ - { - "name": "review_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Review Id" - } - } + "Scraping" ], + "summary": "Detect if a site is blocking the scraper", + "description": "Detect what kind of anti-bot protection a site is using.\n\nReturns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.\nUseful for debugging scraping issues.", + "operationId": "detect_block_v1_detect_block_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Body_review_approve_v1_review__review_id__approve_post" + "type": "string", + "title": "Url" } } - } + }, + "required": true }, "responses": { "200": { @@ -4775,9 +384,9 @@ "content": { "application/json": { "schema": { - "type": "object", "additionalProperties": true, - "title": "Response Review Approve V1 Review Review Id Approve Post" + "type": "object", + "title": "Response Detect Block V1 Detect Block Post" } } } @@ -4795,33 +404,23 @@ } } }, - "/v1/review/{review_id}/reject": { + "/v1/capture/lazy": { "post": { "tags": [ - "Review" - ], - "summary": "Reject a review item", - "description": "Reject a review item, blocking data delivery.", - "operationId": "review_reject_v1_review__review_id__reject_post", - "parameters": [ - { - "name": "review_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Review Id" - } - } + "Scraping" ], + "summary": "Detect and handle lazy-loaded content", + "description": "Detect lazy loading and infinite scroll patterns on a page.\n\nOptionally generate JS to auto-scroll and load all content.", + "operationId": "detect_lazy_content_v1_capture_lazy_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Body_review_reject_v1_review__review_id__reject_post" + "$ref": "#/components/schemas/Body_detect_lazy_content_v1_capture_lazy_post" } } - } + }, + "required": true }, "responses": { "200": { @@ -4829,9 +428,9 @@ "content": { "application/json": { "schema": { - "type": "object", "additionalProperties": true, - "title": "Response Review Reject V1 Review Review Id Reject Post" + "type": "object", + "title": "Response Detect Lazy Content V1 Capture Lazy Post" } } } @@ -4849,714 +448,111 @@ } } }, - "/v1/reviews": { - "get": { + "/v1/crawl": { + "post": { "tags": [ - "Review" + "Scraping" ], - "summary": "List reviews in the queue", - "description": "List reviews, optionally filtered by status (pending/approved/rejected).", - "operationId": "list_reviews_v1_reviews_get", + "summary": "Crawl multiple pages from a URL", + "description": "Crawl multiple pages from a URL. Supports async webhooks.", + "operationId": "crawl_v1_crawl_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CrawlRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Crawl V1 Crawl Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/map": { + "post": { + "tags": [ + "Scraping" + ], + "summary": "Discover URLs on a site", + "description": "Discover URLs on a site.", + "operationId": "map_pages_v1_map_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MapRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Map Pages V1 Map Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/batch": { + "post": { + "tags": [ + "Scraping" + ], + "summary": "Scrape multiple URLs in parallel", + "description": "Scrape multiple URLs in parallel. Firecrawl charges extra for batch.", + "operationId": "batch_scrape_v1_batch_post", "parameters": [ { - "name": "status", + "name": "timeout", "in": "query", "required": false, "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Status" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response List Reviews V1 Reviews Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/review/{review_id}": { - "get": { - "tags": [ - "Review" - ], - "summary": "Get review details", - "description": "Get full details of a review item including the data payload.", - "operationId": "get_review_v1_review__review_id__get", - "parameters": [ - { - "name": "review_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Review Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Review V1 Review Review Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/extract-with-review": { - "post": { - "tags": [ - "Review" - ], - "summary": "Extract with automatic human review routing", - "description": "Extract data with automatic quality check and human review routing.\n\nHigh-confidence results are auto-approved.\nLow-confidence results are auto-rejected.\nMedium-confidence results go to the human review queue with Slack notification.", - "operationId": "extract_with_review_v1_extract_with_review_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_extract_with_review_v1_extract_with_review_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Extract With Review V1 Extract With Review Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/commerce/sync": { - "post": { - "tags": [ - "Commerce" - ], - "summary": "Sync products to WooCommerce or Shopify", - "description": "Sync scraped products to your e-commerce platform.\n\nPlatforms:\n- woocommerce: credentials = {\"wp_url\": \"...\", \"consumer_key\": \"...\", \"consumer_secret\": \"...\"}\n- shopify: credentials = {\"shop_url\": \"...\", \"access_token\": \"...\"}\n\nProducts are imported as drafts for review before publishing.", - "operationId": "sync_commerce_v1_commerce_sync_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_sync_commerce_v1_commerce_sync_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Sync Commerce V1 Commerce Sync Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/commerce/platforms": { - "get": { - "tags": [ - "Commerce" - ], - "summary": "List supported commerce platforms", - "description": "List supported e-commerce platforms and their credential requirements.", - "operationId": "list_commerce_platforms_v1_commerce_platforms_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Commerce Platforms V1 Commerce Platforms Get" - } - } - } - } - } - } - }, - "/v1/crm/sync": { - "post": { - "tags": [ - "CRM" - ], - "summary": "Sync scraped data to CRM", - "description": "Sync scraped data to your CRM.\n\nPlatforms:\n- salesforce: credentials={\"instance_url\": \"...\", \"access_token\": \"...\"}\n- hubspot: credentials={\"api_key\": \"...\"}\n- pipedrive: credentials={\"api_token\": \"...\", \"domain\": \"...\"}\n- close: credentials={\"api_key\": \"...\"}\n\nObject types vary by platform:\n- Salesforce: Lead, Contact, Account, Opportunity\n- HubSpot: contacts, companies, deals\n- Pipedrive: person, organization, deal, lead\n- Close: lead, contact", - "operationId": "sync_crm_v1_crm_sync_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_sync_crm_v1_crm_sync_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Sync Crm V1 Crm Sync Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/crm/platforms": { - "get": { - "tags": [ - "CRM" - ], - "summary": "List supported CRM platforms", - "description": "List supported CRM platforms and their credential requirements.", - "operationId": "list_crm_platforms_v1_crm_platforms_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Crm Platforms V1 Crm Platforms Get" - } - } - } - } - } - } - }, - "/v1/pipelines/steps": { - "get": { - "tags": [ - "Pipelines" - ], - "summary": "List all available pipeline step types", - "description": "List all available step types for the visual pipeline builder.\n\nEach step type includes: name, icon, description, required inputs with types,\nand expected outputs. A UI renders these as drag-and-drop blocks.", - "operationId": "list_step_types_v1_pipelines_steps_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Step Types V1 Pipelines Steps Get" - } - } - } - } - } - } - }, - "/v1/pipelines/validate": { - "post": { - "tags": [ - "Pipelines" - ], - "summary": "Validate a pipeline definition", - "description": "Validate a pipeline definition for correctness.", - "operationId": "validate_pipeline_endpoint_v1_pipelines_validate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Pipeline" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Validate Pipeline Endpoint V1 Pipelines Validate Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/pipelines/run": { - "post": { - "tags": [ - "Pipelines" - ], - "summary": "Execute a pipeline", - "description": "Execute a pipeline definition.\n\nSteps run sequentially. Each step's output is available as\n{{step_id.output_key}} in subsequent step templates.\n\nExample pipeline:\n{\n \"name\": \"Monitor Competitor Pricing\",\n \"steps\": [\n {\"id\": \"scrape_amazon\", \"type\": \"scrape\", \"inputs\": {\"url\": \"https://amazon.com/product\"}},\n {\"id\": \"extract\", \"type\": \"extract_css\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"schema\": {...}}},\n {\"id\": \"quality\", \"type\": \"quality_check\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"data\": \"{{extract.items}}\"}},\n {\"id\": \"notify\", \"type\": \"send_slack\", \"inputs\": {\"webhook_url\": \"https://hooks.slack.com/...\", \"message\": \"{{quality.quality_score}}\"}},\n ]\n}", - "operationId": "run_pipeline_endpoint_v1_pipelines_run_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_run_pipeline_endpoint_v1_pipelines_run_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Run Pipeline Endpoint V1 Pipelines Run Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/pipelines/save": { - "post": { - "tags": [ - "Pipelines" - ], - "summary": "Save a pipeline definition", - "description": "Save a pipeline definition for later use.", - "operationId": "save_pipeline_endpoint_v1_pipelines_save_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Pipeline" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Save Pipeline Endpoint V1 Pipelines Save Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/pipelines": { - "get": { - "tags": [ - "Pipelines" - ], - "summary": "List saved pipelines", - "description": "List all saved pipeline definitions.", - "operationId": "list_pipelines_endpoint_v1_pipelines_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Pipelines Endpoint V1 Pipelines Get" - } - } - } - } - } - } - }, - "/v1/pipelines/{pipeline_id}": { - "get": { - "tags": [ - "Pipelines" - ], - "summary": "Get a saved pipeline", - "description": "Get a saved pipeline definition by ID.", - "operationId": "get_pipeline_endpoint_v1_pipelines__pipeline_id__get", - "parameters": [ - { - "name": "pipeline_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Pipeline Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Pipeline Endpoint V1 Pipelines Pipeline Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "delete": { - "tags": [ - "Pipelines" - ], - "summary": "Delete a saved pipeline", - "description": "Delete a saved pipeline definition.", - "operationId": "delete_pipeline_endpoint_v1_pipelines__pipeline_id__delete", - "parameters": [ - { - "name": "pipeline_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Pipeline Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Delete Pipeline Endpoint V1 Pipelines Pipeline Id Delete" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/agency/create": { - "post": { - "tags": [ - "Agency" - ], - "summary": "Create a white-label agency profile", - "description": "Create a white-label agency profile for reselling Pry.\n\nAgencies get:\n- Custom branding (colors, logo, domain)\n- Client management with sub-accounts\n- Usage analytics and quota management\n- API key management for each client", - "operationId": "create_agency_endpoint_v1_agency_create_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_create_agency_endpoint_v1_agency_create_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Create Agency Endpoint V1 Agency Create Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/agency/{agency_id}": { - "get": { - "tags": [ - "Agency" - ], - "summary": "Get agency profile", - "description": "Get agency profile details.", - "operationId": "get_agency_endpoint_v1_agency__agency_id__get", - "parameters": [ - { - "name": "agency_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Agency Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Agency Endpoint V1 Agency Agency Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/agency/{agency_id}/branding": { - "put": { - "tags": [ - "Agency" - ], - "summary": "Update agency branding", - "description": "Update white-label branding for an agency.", - "operationId": "update_branding_v1_agency__agency_id__branding_put", - "parameters": [ - { - "name": "agency_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Agency Id" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_update_branding_v1_agency__agency_id__branding_put" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Update Branding V1 Agency Agency Id Branding Put" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/agency/{agency_id}/clients": { - "post": { - "tags": [ - "Agency" - ], - "summary": "Create a client sub-account", - "description": "Create a client sub-account under an agency.\n\nEach client gets their own API key and usage quota.", - "operationId": "create_client_endpoint_v1_agency__agency_id__clients_post", - "parameters": [ - { - "name": "agency_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Agency Id" + "type": "integer", + "default": 30, + "title": "Timeout" } } ], @@ -5565,7 +561,11 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Body_create_client_endpoint_v1_agency__agency_id__clients_post" + "type": "array", + "items": { + "type": "string" + }, + "title": "Urls" } } } @@ -5578,474 +578,7 @@ "schema": { "type": "object", "additionalProperties": true, - "title": "Response Create Client Endpoint V1 Agency Agency Id Clients Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - }, - "get": { - "tags": [ - "Agency" - ], - "summary": "List agency clients", - "description": "List all clients under an agency.", - "operationId": "list_clients_endpoint_v1_agency__agency_id__clients_get", - "parameters": [ - { - "name": "agency_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Agency Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response List Clients Endpoint V1 Agency Agency Id Clients Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/agency/{agency_id}/analytics": { - "get": { - "tags": [ - "Agency" - ], - "summary": "Get agency usage analytics", - "description": "Get aggregate usage analytics for an agency.", - "operationId": "get_analytics_v1_agency__agency_id__analytics_get", - "parameters": [ - { - "name": "agency_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Agency Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Analytics V1 Agency Agency Id Analytics Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/client/{client_id}/quota": { - "get": { - "tags": [ - "Agency" - ], - "summary": "Check client quota usage", - "description": "Check a client's current quota usage and remaining capacity.", - "operationId": "check_quota_v1_client__client_id__quota_get", - "parameters": [ - { - "name": "client_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Client Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Check Quota V1 Client Client Id Quota Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/seo/analyze": { - "post": { - "tags": [ - "SEO" - ], - "summary": "Analyze SEO elements from a URL", - "description": "Analyze all SEO elements from a URL.\n\nReturns: title, meta description, keywords, headings (H1/H2),\ncanonical, OG tags, Twitter cards, word count, link counts,\nschema markup, hreflang tags.", - "operationId": "seo_analyze_v1_seo_analyze_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Seo Analyze V1 Seo Analyze Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/seo/track": { - "post": { - "tags": [ - "SEO" - ], - "summary": "Track SEO changes since last scan", - "description": "Track SEO changes since the last scan of this URL.\n\nCompares current SEO elements to previous snapshot and reports\nwhat changed (title, description, headings, etc.).", - "operationId": "seo_track_v1_seo_track_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Seo Track V1 Seo Track Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/seo/keywords": { - "post": { - "tags": [ - "SEO" - ], - "summary": "Analyze keyword presence in URL content", - "description": "Analyze which keywords a URL targets.\n\nChecks each keyword for:\n- Presence in title tag\n- Presence in H1 headings\n- Presence in meta description\n- Frequency in body content\n- Keyword density percentage", - "operationId": "seo_keywords_v1_seo_keywords_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_seo_keywords_v1_seo_keywords_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Seo Keywords V1 Seo Keywords Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/reports/generate": { - "post": { - "tags": [ - "Reports" - ], - "summary": "Generate a white-label report from scraped data", - "description": "Generate a white-label report from scraped data.\n\nReport types:\n- competitive_analysis: Competitor pricing and activity overview\n- price_monitor: Product price change tracking with visual indicators\n- seo_audit: SEO element analysis with change detection\n- content_tracker: Content change monitoring across pages\n\nBranding (optional): {\"agency_name\": \"...\", \"brand_color\": \"#hex\", \"logo_url\": \"...\"}", - "operationId": "generate_report_endpoint_v1_reports_generate_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_generate_report_endpoint_v1_reports_generate_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Generate Report Endpoint V1 Reports Generate Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/reports": { - "get": { - "tags": [ - "Reports" - ], - "summary": "List generated reports", - "description": "List all previously generated reports.", - "operationId": "list_reports_endpoint_v1_reports_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Reports Endpoint V1 Reports Get" - } - } - } - } - } - } - }, - "/v1/report/{report_id}": { - "get": { - "tags": [ - "Reports" - ], - "summary": "Get a generated report", - "description": "Get the HTML content of a generated report.", - "operationId": "get_report_v1_report__report_id__get", - "parameters": [ - { - "name": "report_id", - "in": "path", - "required": true, - "schema": { - "type": "string", - "title": "Report Id" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "title": "Response Get Report V1 Report Report Id Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/enrich": { - "post": { - "tags": [ - "Enrichment" - ], - "summary": "Enrich scraped data with company info, tech stack, social profiles", - "description": "Enrich a URL with supplemental business intelligence.\n\nReturns:\n- Tech stack detection (CMS, framework, CDN, analytics, payments)\n- Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.)\n- Company information (email, phone, address, founded year, team size)", - "operationId": "enrich_data_v1_enrich_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_enrich_data_v1_enrich_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Enrich Data V1 Enrich Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/enrich/tech-stack": { - "post": { - "tags": [ - "Enrichment" - ], - "summary": "Detect technologies used on a website", - "description": "Detect what technologies a website is built with.\n\nDetects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails),\nfrontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar),\npayments (Stripe, PayPal).", - "operationId": "detect_tech_v1_enrich_tech_stack_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Url" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Detect Tech V1 Enrich Tech Stack Post" + "title": "Response Batch Scrape V1 Batch Post" } } } @@ -6176,9 +709,58 @@ } } }, + "/v1/templates/batch": { + "post": { + "tags": [ + "Templates" + ], + "summary": "Execute multiple templates/URLs in parallel", + "description": "Execute multiple scraper templates against multiple URLs in parallel.\n\nRuns up to 20 template executions concurrently. Each result preserves the\noriginal template_id and url for correlation.", + "operationId": "batch_execute_v1_templates_batch_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/BatchItem" + }, + "type": "array", + "title": "Body" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Batch Execute V1 Templates Batch Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v1/ai/gpt-manifest": { "get": { "tags": [ + "AI", "AI" ], "summary": "Get GPT Action manifest for ChatGPT", @@ -6199,6 +781,7 @@ "/v1/ai/mcp-config": { "get": { "tags": [ + "AI", "AI" ], "summary": "Get MCP server config for Claude/Cursor", @@ -6216,689 +799,10 @@ } } }, - "/v1/referrals/catalog": { - "get": { - "tags": [ - "Referrals" - ], - "summary": "Get all available referral/affiliate programs", - "description": "List all referral programs Pry supports.\n\n60+ providers across categories: LLM, hosting, domains, CDN, email,\nmonitoring, proxies, voice, media, devtools, search, CAPTCHA.", - "operationId": "get_referral_catalog_v1_referrals_catalog_get", - "parameters": [ - { - "name": "category", - "in": "query", - "required": false, - "schema": { - "type": "string", - "default": "", - "title": "Category" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Referral Catalog V1 Referrals Catalog Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/referrals/stats": { - "get": { - "tags": [ - "Referrals" - ], - "summary": "Get referral click and conversion stats", - "description": "Get referral tracking statistics for the last N days.", - "operationId": "get_referral_stats_v1_referrals_stats_get", - "parameters": [ - { - "name": "days_back", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 30, - "title": "Days Back" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Get Referral Stats V1 Referrals Stats Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/referrals/click": { - "post": { - "tags": [ - "Referrals" - ], - "summary": "Record a referral link click", - "description": "Record when a user clicks a referral link. Returns tracking ID.", - "operationId": "record_referral_click_v1_referrals_click_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_record_referral_click_v1_referrals_click_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Record Referral Click V1 Referrals Click Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/referrals/convert": { - "post": { - "tags": [ - "Referrals" - ], - "summary": "Record a referral conversion", - "description": "Record that a referral click resulted in a conversion.", - "operationId": "record_referral_conversion_v1_referrals_convert_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_record_referral_conversion_v1_referrals_convert_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Record Referral Conversion V1 Referrals Convert Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/x402/pricing": { - "get": { - "tags": [ - "x402" - ], - "summary": "Get x402 pricing for all paid operations", - "description": "Get the price list for pay-per-scrape operations.", - "operationId": "x402_pricing_v1_x402_pricing_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response X402 Pricing V1 X402 Pricing Get" - } - } - } - } - } - } - }, - "/v1/x402/payment": { - "post": { - "tags": [ - "x402" - ], - "summary": "Create a x402 payment request", - "description": "Create a x402 payment request for a paid operation.\n\nReturns payment details (wallet, amount, asset) for the client to pay.", - "operationId": "x402_payment_request_v1_x402_payment_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_x402_payment_request_v1_x402_payment_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response X402 Payment Request V1 X402 Payment Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/x402/verify": { - "post": { - "tags": [ - "x402" - ], - "summary": "Verify a x402 payment was settled", - "description": "Verify a x402 payment has been settled on-chain via facilitator router.", - "operationId": "x402_verify_payment_v1_x402_verify_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_x402_verify_payment_v1_x402_verify_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response X402 Verify Payment V1 X402 Verify Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/x402/require-payment": { - "post": { - "tags": [ - "x402" - ], - "summary": "Generate a 402 Payment Required response", - "description": "Generate a 402 Payment Required response for a paid endpoint.", - "operationId": "x402_require_payment_v1_x402_require_payment_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_x402_require_payment_v1_x402_require_payment_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response X402 Require Payment V1 X402 Require Payment Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/x402/batch-payment": { - "post": { - "tags": [ - "x402" - ], - "summary": "Create a batch x402 payment", - "description": "Create a single x402 payment covering multiple operations.\n\nReturns a PaymentRequired body with the combined amount. After paying,\nsubmit the tx to POST /v1/x402/batch-verify.", - "operationId": "x402_batch_payment_v1_x402_batch_payment_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Payload" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response X402 Batch Payment V1 X402 Batch Payment Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/x402/batch-verify": { - "post": { - "tags": [ - "x402" - ], - "summary": "Verify a batch x402 payment", - "description": "Verify the on-chain payment for a batch and mark it paid.", - "operationId": "x402_batch_verify_v1_x402_batch_verify_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Payload" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response X402 Batch Verify V1 X402 Batch Verify Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/proxy/providers": { - "get": { - "tags": [ - "Proxy" - ], - "summary": "List all available proxy providers", - "description": "List free + premium proxy providers with affiliate details.", - "operationId": "list_proxy_providers_v1_proxy_providers_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response List Proxy Providers V1 Proxy Providers Get" - } - } - } - } - } - } - }, - "/v1/proxy/signup": { - "post": { - "tags": [ - "Proxy" - ], - "summary": "Open affiliate signup link for a provider", - "description": "Get the affiliate signup URL for a proxy provider.\n\nRecords a click for revenue tracking. The user can then sign up at that URL\nand come back to configure credentials.", - "operationId": "proxy_signup_v1_proxy_signup_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Provider" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Proxy Signup V1 Proxy Signup Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/proxy/configure": { - "post": { - "tags": [ - "Proxy" - ], - "summary": "Configure proxy credentials", - "description": "Configure credentials for a premium proxy provider.\n\nAfter signing up via /v1/proxy/signup, the user provides their credentials here.", - "operationId": "proxy_configure_v1_proxy_configure_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Body_proxy_configure_v1_proxy_configure_post" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Proxy Configure V1 Proxy Configure Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/proxy/test": { - "get": { - "tags": [ - "Proxy" - ], - "summary": "Test the active proxy", - "description": "Test the currently configured proxy and return its public IP.", - "operationId": "proxy_test_v1_proxy_test_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Proxy Test V1 Proxy Test Get" - } - } - } - } - } - } - }, - "/v1/proxy/status": { - "get": { - "tags": [ - "Proxy" - ], - "summary": "Get current proxy status", - "description": "Get current proxy configuration and available providers.", - "operationId": "proxy_status_v1_proxy_status_get", - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Proxy Status V1 Proxy Status Get" - } - } - } - } - } - } - }, - "/v1/proxy/recommend": { - "post": { - "tags": [ - "Proxy" - ], - "summary": "Get proxy recommendation after a block", - "description": "After a scrape fails with anti-bot detection, get a recommendation\nfor which premium proxy provider to sign up with.", - "operationId": "proxy_recommend_v1_proxy_recommend_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "string", - "title": "Last Error", - "default": "" - } - } - } - }, - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "additionalProperties": true, - "type": "object", - "title": "Response Proxy Recommend V1 Proxy Recommend Post" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v1/proxy/clicks": { - "get": { - "tags": [ - "Proxy" - ], - "summary": "Get recent proxy referral clicks", - "description": "Get recent proxy referral clicks for revenue tracking.", - "operationId": "proxy_clicks_v1_proxy_clicks_get", - "parameters": [ - { - "name": "days_back", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "default": 30, - "title": "Days Back" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": true, - "title": "Response Proxy Clicks V1 Proxy Clicks Get" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v1/tls/impersonate": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Fetch a URL with TLS fingerprint impersonation", @@ -6943,6 +847,7 @@ "/v1/tls/rotate": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Try multiple browser fingerprints until one succeeds", @@ -6987,6 +892,7 @@ "/v1/camoufox/fetch": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Fetch with Camoufox anti-detection Firefox", @@ -7031,6 +937,7 @@ "/v1/graphql/discover": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Discover GraphQL endpoints for a site", @@ -7076,6 +983,7 @@ "/v1/graphql/introspect": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Run GraphQL introspection on an endpoint", @@ -7121,6 +1029,7 @@ "/v1/graphql/query": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Execute a GraphQL query", @@ -7165,6 +1074,7 @@ "/v1/schema/extract": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Extract Schema.org structured data from a page", @@ -7210,6 +1120,7 @@ "/v1/schema/extract-html": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Extract Schema.org structured data from raw HTML", @@ -7255,6 +1166,7 @@ "/v1/ws/scrape": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Scrape data from a WebSocket endpoint", @@ -7299,6 +1211,7 @@ "/v1/sse/scrape": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Scrape data from a Server-Sent Events endpoint", @@ -7343,6 +1256,7 @@ "/v1/cookies/warm": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Warm cookies for a domain by browsing legitimate pages", @@ -7387,6 +1301,7 @@ "/v1/cookies/sessions": { "get": { "tags": [ + "Advanced", "Advanced" ], "summary": "List all warmed cookie sessions", @@ -7410,6 +1325,7 @@ "/v1/pdf/extract": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Extract tables and text from a PDF", @@ -7454,6 +1370,7 @@ "/v1/ocr/extract": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Extract text from an image using Tesseract", @@ -7497,6 +1414,7 @@ "/v1/dedup/check": { "post": { "tags": [ + "Advanced", "Advanced" ], "summary": "Check if content is a near-duplicate using SimHash", @@ -7541,6 +1459,7 @@ "/v1/behavior/simulate": { "get": { "tags": [ + "Advanced", "Advanced" ], "summary": "Generate human-like behavior patterns for testing", @@ -7584,19 +1503,20 @@ } } }, - "/v1/x402/pay": { + "/v1/agency/create": { "post": { "tags": [ - "x402" + "Agency", + "Agency" ], - "summary": "Process x402 payment and get access token", - "description": "Process an x402 payment and return an access token.\n\nFlow:\n1. User gets 402 from a paid endpoint\n2. User sends USDC to the wallet in the 402 response\n3. User calls this endpoint with the tx_hash\n4. Pry verifies the transaction through the facilitator router\n5. Returns access token (payment_id) to use in X-Payment-Hash header", - "operationId": "x402_pay_v1_x402_pay_post", + "summary": "Create a white-label agency profile", + "description": "Create a white-label agency profile for reselling Pry.\n\nAgencies get:\n- Custom branding (colors, logo, domain)\n- Client management with sub-accounts\n- Usage analytics and quota management\n- API key management for each client", + "operationId": "create_agency_endpoint_v1_agency_create_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Body_x402_pay_v1_x402_pay_post" + "$ref": "#/components/schemas/Body_create_agency_endpoint_v1_agency_create_post" } } }, @@ -7610,7 +1530,2979 @@ "schema": { "additionalProperties": true, "type": "object", - "title": "Response X402 Pay V1 X402 Pay Post" + "title": "Response Create Agency Endpoint V1 Agency Create Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/agency/{agency_id}": { + "get": { + "tags": [ + "Agency", + "Agency" + ], + "summary": "Get agency profile", + "description": "Get agency profile details.", + "operationId": "get_agency_endpoint_v1_agency__agency_id__get", + "parameters": [ + { + "name": "agency_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Agency Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Agency Endpoint V1 Agency Agency Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/agency/{agency_id}/branding": { + "put": { + "tags": [ + "Agency", + "Agency" + ], + "summary": "Update agency branding", + "description": "Update white-label branding for an agency.", + "operationId": "update_branding_v1_agency__agency_id__branding_put", + "parameters": [ + { + "name": "agency_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Agency Id" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_update_branding_v1_agency__agency_id__branding_put" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Update Branding V1 Agency Agency Id Branding Put" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/agency/{agency_id}/clients": { + "post": { + "tags": [ + "Agency", + "Agency" + ], + "summary": "Create a client sub-account", + "description": "Create a client sub-account under an agency.\n\nEach client gets their own API key and usage quota.", + "operationId": "create_client_endpoint_v1_agency__agency_id__clients_post", + "parameters": [ + { + "name": "agency_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Agency Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_create_client_endpoint_v1_agency__agency_id__clients_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Create Client Endpoint V1 Agency Agency Id Clients Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "get": { + "tags": [ + "Agency", + "Agency" + ], + "summary": "List agency clients", + "description": "List all clients under an agency.", + "operationId": "list_clients_endpoint_v1_agency__agency_id__clients_get", + "parameters": [ + { + "name": "agency_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Agency Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response List Clients Endpoint V1 Agency Agency Id Clients Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/agency/{agency_id}/analytics": { + "get": { + "tags": [ + "Agency", + "Agency" + ], + "summary": "Get agency usage analytics", + "description": "Get aggregate usage analytics for an agency.", + "operationId": "get_analytics_v1_agency__agency_id__analytics_get", + "parameters": [ + { + "name": "agency_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Agency Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Analytics V1 Agency Agency Id Analytics Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/client/{client_id}/quota": { + "get": { + "tags": [ + "Agency", + "Agency" + ], + "summary": "Check client quota usage", + "description": "Check a client's current quota usage and remaining capacity.", + "operationId": "check_quota_v1_client__client_id__quota_get", + "parameters": [ + { + "name": "client_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Client Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Check Quota V1 Client Client Id Quota Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/alert/send": { + "post": { + "tags": [ + "Alerts", + "Alerts" + ], + "summary": "Send an alert to any channel", + "description": "Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email.\n\nConfig varies by channel:\n- slack: {\"webhook_url\": \"...\"}\n- discord: {\"webhook_url\": \"...\"}\n- teams: {\"webhook_url\": \"...\"}\n- telegram: {\"bot_token\": \"...\", \"chat_id\": \"...\"}\n- sms: {\"phone\": \"+1234567890\", \"twilio_sid\": \"...\", \"twilio_token\": \"...\", \"twilio_from\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"smtp_host\": \"...\", \"smtp_user\": \"...\"}", + "operationId": "send_alert_endpoint_v1_alert_send_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_send_alert_endpoint_v1_alert_send_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Send Alert Endpoint V1 Alert Send Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/alert/channels": { + "get": { + "tags": [ + "Alerts", + "Alerts" + ], + "summary": "List supported alert channels", + "description": "List all supported alert channels and their config requirements.", + "operationId": "list_channels_v1_alert_channels_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Channels V1 Alert Channels Get" + } + } + } + } + } + } + }, + "/v1/compare": { + "post": { + "tags": [ + "Analysis", + "Analysis" + ], + "summary": "Compare content of two URLs", + "description": "Scrape two URLs and compare their content. Shows additions, deletions, changes.", + "operationId": "compare_v1_compare_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_compare_v1_compare_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Compare V1 Compare Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/watch": { + "post": { + "tags": [ + "Analysis", + "Analysis" + ], + "summary": "Watch a page for changes", + "description": "Watch a page for changes. Accepts a webhook URL for notification.\nFirst call registers the watch, subsequent calls compare and notify.\nFirecrawl charges $99/mo for this feature.", + "operationId": "watch_page_v1_watch_post", + "parameters": [ + { + "name": "interval", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 3600, + "title": "Interval" + } + }, + { + "name": "selector", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "", + "title": "Selector" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_watch_page_v1_watch_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Watch Page V1 Watch Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/summarize": { + "post": { + "tags": [ + "Analysis", + "Analysis" + ], + "summary": "AI summarize scraped content", + "description": "Scrape + AI summarize using local Ollama. Free, private.", + "operationId": "summarize_v1_summarize_post", + "parameters": [ + { + "name": "max_words", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 100, + "title": "Max Words" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Summarize V1 Summarize Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/diff": { + "post": { + "tags": [ + "Analysis", + "Analysis" + ], + "summary": "Track page changes over time", + "description": "Track page changes over time. Returns diff from last scrape.", + "operationId": "diff_v1_diff_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_diff_v1_diff_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Diff V1 Diff Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/categorize": { + "post": { + "tags": [ + "Analysis", + "Analysis" + ], + "summary": "AI-categorize scraped content", + "description": "AI-categorize scraped content into topic tags.", + "operationId": "categorize_v1_categorize_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Categorize V1 Categorize Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/suggest": { + "post": { + "tags": [ + "Analysis", + "Analysis" + ], + "summary": "AI-suggest schema fields for a URL", + "operationId": "suggest_schema_v1_suggest_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Data" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Suggest Schema V1 Suggest Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/automate": { + "post": { + "tags": [ + "Automation", + "Automation" + ], + "summary": "Execute browser automation steps", + "description": "Execute browser automation steps. Sessions persist for login flows.", + "operationId": "automate_v1_automate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AutomateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Automate V1 Automate Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/screenshot": { + "post": { + "tags": [ + "Automation", + "Automation" + ], + "summary": "Take a screenshot of a URL", + "description": "Take a screenshot of a URL. Returns base64 PNG.", + "operationId": "screenshot_v1_screenshot_post", + "parameters": [ + { + "name": "session_id", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Session Id" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_screenshot_v1_screenshot_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Screenshot V1 Screenshot Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/batch-file": { + "post": { + "tags": [ + "Batch", + "Batch" + ], + "summary": "Batch scrape URLs from a file with templates", + "operationId": "batch_from_file_v1_batch_file_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BatchFileRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Batch From File V1 Batch File Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/crm/sync": { + "post": { + "tags": [ + "CRM", + "CRM" + ], + "summary": "Sync scraped data to CRM", + "description": "Sync scraped data to your CRM.\n\nPlatforms:\n- salesforce: credentials={\"instance_url\": \"...\", \"access_token\": \"...\"}\n- hubspot: credentials={\"api_key\": \"...\"}\n- pipedrive: credentials={\"api_token\": \"...\", \"domain\": \"...\"}\n- close: credentials={\"api_key\": \"...\"}\n\nObject types vary by platform:\n- Salesforce: Lead, Contact, Account, Opportunity\n- HubSpot: contacts, companies, deals\n- Pipedrive: person, organization, deal, lead\n- Close: lead, contact", + "operationId": "sync_crm_v1_crm_sync_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_sync_crm_v1_crm_sync_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Sync Crm V1 Crm Sync Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/crm/platforms": { + "get": { + "tags": [ + "CRM", + "CRM" + ], + "summary": "List supported CRM platforms", + "description": "List supported CRM platforms and their credential requirements.", + "operationId": "list_crm_platforms_v1_crm_platforms_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Crm Platforms V1 Crm Platforms Get" + } + } + } + } + } + } + }, + "/v1/breaker/status": { + "post": { + "tags": [ + "Circuit Breaker", + "Circuit Breaker" + ], + "summary": "Get circuit breaker status", + "operationId": "breaker_status_v1_breaker_status_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Breaker Status V1 Breaker Status Post" + } + } + } + } + } + } + }, + "/v1/breaker/reset": { + "post": { + "tags": [ + "Circuit Breaker", + "Circuit Breaker" + ], + "summary": "Reset circuit breaker for a domain", + "operationId": "breaker_reset_v1_breaker_reset_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Domain", + "default": "" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Breaker Reset V1 Breaker Reset Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/commerce/sync": { + "post": { + "tags": [ + "Commerce", + "Commerce" + ], + "summary": "Sync products to WooCommerce or Shopify", + "description": "Sync scraped products to your e-commerce platform.\n\nPlatforms:\n- woocommerce: credentials = {\"wp_url\": \"...\", \"consumer_key\": \"...\", \"consumer_secret\": \"...\"}\n- shopify: credentials = {\"shop_url\": \"...\", \"access_token\": \"...\"}\n\nProducts are imported as drafts for review before publishing.", + "operationId": "sync_commerce_v1_commerce_sync_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_sync_commerce_v1_commerce_sync_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Sync Commerce V1 Commerce Sync Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/commerce/platforms": { + "get": { + "tags": [ + "Commerce", + "Commerce" + ], + "summary": "List supported commerce platforms", + "description": "List supported e-commerce platforms and their credential requirements.", + "operationId": "list_commerce_platforms_v1_commerce_platforms_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Commerce Platforms V1 Commerce Platforms Get" + } + } + } + } + } + } + }, + "/v1/compliance/check": { + "get": { + "tags": [ + "Compliance", + "Compliance" + ], + "summary": "Get compliance check documentation", + "description": "Get information about the compliance check endpoint.", + "operationId": "compliance_docs_v1_compliance_check_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Compliance Docs V1 Compliance Check Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "Compliance", + "Compliance" + ], + "summary": "Run full compliance check on a URL", + "description": "Run a full legal compliance check on a target URL.\n\nAnalyzes:\n- robots.txt crawl permissions\n- Terms of Service classification (restrictive/permissive/moderate)\n- Jurisdiction detection (GDPR, CCPA, LGPD)\n- Sensitive data detection (PII, financial, health, contact)\n\nReturns a green/yellow/red risk score with recommendations.", + "operationId": "compliance_check_v1_compliance_check_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Compliance Check V1 Compliance Check Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/config": { + "get": { + "tags": [ + "Config", + "Config" + ], + "summary": "Get current Pry configuration", + "description": "Get current Pry configuration.", + "operationId": "get_config_v1_config_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Get Config V1 Config Get" + } + } + } + } + } + }, + "post": { + "tags": [ + "Config", + "Config" + ], + "summary": "Update Pry configuration at runtime", + "description": "Update Pry configuration at runtime.", + "operationId": "update_config_v1_config_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Updates" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Update Config V1 Config Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/config/profile/tor": { + "post": { + "tags": [ + "Config", + "Config" + ], + "summary": "Enable Tor routing for all requests", + "description": "Enable Tor routing for all requests.", + "operationId": "enable_tor_v1_config_profile_tor_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Enable Tor V1 Config Profile Tor Post" + } + } + } + } + } + } + }, + "/v1/costing/dashboard": { + "get": { + "tags": [ + "Costing", + "Costing" + ], + "summary": "Get cost analytics dashboard", + "description": "Get the full cost analytics dashboard.\n\nIncludes:\n- Monthly spend breakdown by operation type\n- Projected end-of-month cost\n- Daily spend for last 7 days\n- Cache efficiency metrics\n- Smart schedule recommendations", + "operationId": "cost_dashboard_v1_costing_dashboard_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Cost Dashboard V1 Costing Dashboard Get" + } + } + } + } + } + } + }, + "/v1/costing/usage": { + "get": { + "tags": [ + "Costing", + "Costing" + ], + "summary": "Get usage breakdown", + "description": "Get detailed usage breakdown for a specific month.", + "operationId": "cost_usage_v1_costing_usage_get", + "parameters": [ + { + "name": "year", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Year" + } + }, + { + "name": "month", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Month" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Cost Usage V1 Costing Usage Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/costing/record": { + "post": { + "tags": [ + "Costing", + "Costing" + ], + "summary": "Record a usage event", + "description": "Record a usage event for cost tracking.\n\nOperations: scrape_direct, scrape_flaresolverr, scrape_playwright,\ncrawl_page, llm_call, vision_call, extraction_css, bandwidth_mb", + "operationId": "record_usage_endpoint_v1_costing_record_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_record_usage_endpoint_v1_costing_record_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Record Usage Endpoint V1 Costing Record Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/costing/costs": { + "post": { + "tags": [ + "Costing", + "Costing" + ], + "summary": "Update per-operation cost table", + "description": "Update the per-operation cost table with custom prices.", + "operationId": "update_costs_v1_costing_costs_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "number" + }, + "type": "object", + "title": "Costs" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Update Costs V1 Costing Costs Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/dashboard": { + "get": { + "tags": [ + "Dashboard", + "Dashboard" + ], + "summary": "Pry health and performance dashboard", + "operationId": "dashboard_dashboard_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/v1/email/scrape": { + "post": { + "tags": [ + "Email", + "Email" + ], + "summary": "Extract data from an email (subject + body)", + "description": "Extract structured data from an email subject and body.\n\nAuto-classifies as: order_confirmation, invoice, receipt,\nshipping_notification, subscription, or other.\n\nExtracts: order numbers, amounts, tracking numbers, dates, addresses.", + "operationId": "scrape_email_v1_email_scrape_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_scrape_email_v1_email_scrape_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Scrape Email V1 Email Scrape Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/email/gmail": { + "post": { + "tags": [ + "Email", + "Email" + ], + "summary": "Fetch and extract data from Gmail inbox", + "description": "Connect to Gmail and extract structured data from emails.\n\nRequires a Gmail OAuth2 access token with scope:\nhttps://www.googleapis.com/auth/gmail.readonly\n\nExtracts order confirmations, invoices, receipts, shipping notifications.", + "operationId": "fetch_gmail_v1_email_gmail_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_fetch_gmail_v1_email_gmail_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Fetch Gmail V1 Email Gmail Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/email/outlook": { + "post": { + "tags": [ + "Email", + "Email" + ], + "summary": "Fetch and extract data from Outlook inbox", + "description": "Connect to Outlook/Office 365 and extract structured data from emails.\n\nRequires a Microsoft Graph OAuth2 access token with scope:\nhttps://graph.microsoft.com/Mail.Read\n\nExtracts order confirmations, invoices, receipts, shipping notifications.", + "operationId": "fetch_outlook_v1_email_outlook_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_fetch_outlook_v1_email_outlook_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Fetch Outlook V1 Email Outlook Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/enrich": { + "post": { + "tags": [ + "Enrichment", + "Enrichment" + ], + "summary": "Enrich scraped data with company info, tech stack, social profiles", + "description": "Enrich a URL with supplemental business intelligence.\n\nReturns:\n- Tech stack detection (CMS, framework, CDN, analytics, payments)\n- Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.)\n- Company information (email, phone, address, founded year, team size)", + "operationId": "enrich_data_v1_enrich_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_enrich_data_v1_enrich_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Enrich Data V1 Enrich Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/enrich/tech-stack": { + "post": { + "tags": [ + "Enrichment", + "Enrichment" + ], + "summary": "Detect technologies used on a website", + "description": "Detect what technologies a website is built with.\n\nDetects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails),\nfrontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar),\npayments (Stripe, PayPal).", + "operationId": "detect_tech_v1_enrich_tech_stack_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Detect Tech V1 Enrich Tech Stack Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/run": { + "post": { + "tags": [ + "Execute", + "Execute" + ], + "summary": "Execute a Pryfile", + "operationId": "run_pryfile_v1_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Path", + "default": "pry.yml" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Run Pryfile V1 Run Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/export": { + "post": { + "tags": [ + "Export", + "Export" + ], + "summary": "Export scraped content in multiple formats", + "description": "Export scraped content in multiple formats: json, csv, txt, rss.\nFirecrawl only does markdown.", + "operationId": "export_data_v1_export_post", + "parameters": [ + { + "name": "format", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "json", + "title": "Format" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Export Data V1 Export Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/extract-table": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Extract HTML tables as structured data", + "description": "Extract HTML tables from a page as structured data.\nFirecrawl doesn't support table extraction at all.", + "operationId": "extract_table_v1_extract_table_post", + "parameters": [ + { + "name": "table_index", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 0, + "title": "Table Index" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Extract Table V1 Extract Table Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/links": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Analyze links on a page", + "description": "Analyze all links on a page \u2014 internal, external, broken, social.\nFirecrawl only has basic map functionality.", + "operationId": "analyze_links_v1_links_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Analyze Links V1 Links Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/seo": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "SEO analysis of a page", + "description": "SEO analysis of a page: title, description, headings, images, keywords, readability.\nFirecrawl has zero SEO features.", + "operationId": "analyze_seo_v1_seo_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Analyze Seo V1 Seo Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/schema": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Extract Schema.org/JSON-LD structured data", + "description": "Extract Schema.org/JSON-LD structured data from a page.", + "operationId": "extract_schema_v1_schema_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Extract Schema V1 Schema Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/emails": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Find email addresses on a page", + "description": "Find all email addresses on a page.", + "operationId": "find_emails_v1_emails_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Find Emails V1 Emails Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/extract": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Extract structured fields from a URL", + "operationId": "extract_stable_v1_extract_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Data" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Extract Stable V1 Extract Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/extract/css": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Extract structured data with CSS selectors (no LLM)", + "description": "Extract structured JSON from a URL using CSS selector schema.\n\nSchema format:\n{\n \"name\": \"products\",\n \"base_selector\": \".product-card\",\n \"fields\": [\n {\"name\": \"title\", \"selector\": \"h3\", \"type\": \"text\"},\n {\"name\": \"price\", \"selector\": \".price\", \"type\": \"text\", \"transform\": \"float\"},\n {\"name\": \"link\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"},\n {\"name\": \"in_stock\", \"selector\": \".stock\", \"type\": \"exists\"},\n ]\n}", + "operationId": "extract_css_v1_extract_css_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_extract_css_v1_extract_css_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Extract Css V1 Extract Css Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/extract/llm": { + "post": { + "tags": [ + "Extraction", + "Extraction" + ], + "summary": "Extract with LLM + chunking strategies", + "description": "Extract structured data using LLM with intelligent chunking.\n\nChunks content by strategy (topic/sentence/regex), optionally filters\nby relevance to query, then extracts from each chunk.", + "operationId": "extract_llm_v1_extract_llm_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_extract_llm_v1_extract_llm_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Extract Llm V1 Extract Llm Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/freshness/check": { + "post": { + "tags": [ + "Freshness", + "Freshness" + ], + "summary": "Check if content has changed since last scrape", + "description": "Check if content has changed since the last scrape.\n\nUses content fingerprinting (SHA256) to detect changes.\nIf no content is provided, does a quick HEAD check instead.", + "operationId": "freshness_check_v1_freshness_check_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_freshness_check_v1_freshness_check_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Freshness Check V1 Freshness Check Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/freshness/frequency": { + "post": { + "tags": [ + "Freshness", + "Freshness" + ], + "summary": "Get adaptive scrape frequency recommendation", + "description": "Get an adaptive scrape frequency recommendation based on content volatility.\n\nVolatile pages (frequent changes) get shorter intervals.\nStable pages get longer intervals to save costs.", + "operationId": "freshness_frequency_v1_freshness_frequency_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_freshness_frequency_v1_freshness_frequency_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Freshness Frequency V1 Freshness Frequency Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/freshness/dashboard": { + "get": { + "tags": [ + "Freshness", + "Freshness" + ], + "summary": "Get content staleness dashboard", + "description": "Get the content staleness dashboard.\n\nShows all tracked URLs, their last check time, age, and whether\nthey're stale (not checked in 24h).", + "operationId": "freshness_dashboard_v1_freshness_dashboard_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Freshness Dashboard V1 Freshness Dashboard Get" + } + } + } + } + } + } + }, + "/v1/gdpr/consent": { + "post": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Record user consent for data processing", + "description": "Record a user's consent for data processing (GDPR Art. 7).\n\nStores: user hash, purpose, timestamp, IP, user agent.\nConsent expires after 365 days.", + "operationId": "record_consent_endpoint_v1_gdpr_consent_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_record_consent_endpoint_v1_gdpr_consent_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Record Consent Endpoint V1 Gdpr Consent Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/gdpr/consent/{user_id}": { + "get": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Check user consent status", + "description": "Check if a user has given valid consent for data processing.", + "operationId": "check_consent_endpoint_v1_gdpr_consent__user_id__get", + "parameters": [ + { + "name": "user_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "User Id" + } + }, + { + "name": "purpose", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "data_collection", + "title": "Purpose" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Check Consent Endpoint V1 Gdpr Consent User Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/gdpr/consent/revoke": { + "post": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Revoke user consent", + "description": "Revoke a user's consent for a processing purpose (GDPR Art. 7(3)).", + "operationId": "revoke_consent_endpoint_v1_gdpr_consent_revoke_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_revoke_consent_endpoint_v1_gdpr_consent_revoke_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Revoke Consent Endpoint V1 Gdpr Consent Revoke Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/gdpr/deletion/request": { + "post": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Request data deletion (right to erasure)", + "description": "Request deletion of all data associated with a user (GDPR Art. 17).\n\nCreates a deletion request that can be executed immediately or\nafter a holding period.", + "operationId": "request_deletion_endpoint_v1_gdpr_deletion_request_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_request_deletion_endpoint_v1_gdpr_deletion_request_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Request Deletion Endpoint V1 Gdpr Deletion Request Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/gdpr/deletion/execute": { + "post": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Execute data deletion request", + "description": "Execute a pending deletion request, removing all user data.", + "operationId": "execute_deletion_endpoint_v1_gdpr_deletion_execute_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Request Id" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Execute Deletion Endpoint V1 Gdpr Deletion Execute Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/gdpr/retention": { + "get": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Get data retention policy", + "description": "Get the current data retention policy.", + "operationId": "get_retention_policy_endpoint_v1_gdpr_retention_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Get Retention Policy Endpoint V1 Gdpr Retention Get" + } + } + } + } + } + } + }, + "/v1/gdpr/retention/apply": { + "post": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Apply retention policy to remove expired data", + "description": "Apply the retention policy, removing all expired data.", + "operationId": "apply_retention_v1_gdpr_retention_apply_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Apply Retention V1 Gdpr Retention Apply Post" + } + } + } + } + } + } + }, + "/v1/gdpr/audit": { + "get": { + "tags": [ + "GDPR", + "GDPR" + ], + "summary": "Get compliance audit log", + "description": "Get the compliance audit log for the specified period.", + "operationId": "get_audit_log_endpoint_v1_gdpr_audit_get", + "parameters": [ + { + "name": "days_back", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 7, + "title": "Days Back" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Audit Log Endpoint V1 Gdpr Audit Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/destinations": { + "get": { + "tags": [ + "Integrations", + "Integrations" + ], + "summary": "List supported data destinations", + "description": "List all supported data destinations and their config requirements.", + "operationId": "list_destinations_v1_destinations_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Destinations V1 Destinations Get" + } + } + } + } + } + } + }, + "/v1/destination/send": { + "post": { + "tags": [ + "Integrations", + "Integrations" + ], + "summary": "Send scraped data to a business destination", + "description": "Send extracted data to a business destination in one click.\n\nDestinations:\n- slack: Send to Slack channel via webhook\n- googlesheets: Write to Google Sheets (requires credentials)\n- airtable: Write to Airtable base (requires API key)\n- email: Send via email (requires SMTP config)\n\nConfig varies by destination:\n- slack: {\"webhook_url\": \"https://hooks.slack.com/...\"}\n- googlesheets: {\"spreadsheet_id\": \"...\", \"credentials_json\": \"...\"}\n- airtable: {\"base_id\": \"...\", \"table_name\": \"Table 1\", \"api_key\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"subject\": \"Data Export\"}", + "operationId": "send_to_destination_v1_destination_send_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_send_to_destination_v1_destination_send_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Send To Destination V1 Destination Send Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/scrape-and-send": { + "post": { + "tags": [ + "Integrations", + "Integrations" + ], + "summary": "Scrape a URL and send to a destination", + "description": "Scrape a URL and send the results to a business destination in one step.\n\nCombines /v1/scrape + /v1/destination/send into a single call.\nPerfect for non-technical users who just want data in their tools.", + "operationId": "scrape_and_send_v1_scrape_and_send_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_scrape_and_send_v1_scrape_and_send_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Scrape And Send V1 Scrape And Send Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/intel/snapshot": { + "post": { + "tags": [ + "Intelligence", + "Intelligence" + ], + "summary": "Record a competitor data snapshot", + "description": "Record a data snapshot for a competitor.\n\nSnapshots are stored with timestamps and used for trend analysis,\nanomaly detection, and report generation.", + "operationId": "record_intel_snapshot_v1_intel_snapshot_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_record_intel_snapshot_v1_intel_snapshot_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Record Intel Snapshot V1 Intel Snapshot Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/intel/snapshots/{competitor_id}": { + "get": { + "tags": [ + "Intelligence", + "Intelligence" + ], + "summary": "Get competitor snapshots", + "description": "Get historical snapshots for a competitor.", + "operationId": "get_intel_snapshots_v1_intel_snapshots__competitor_id__get", + "parameters": [ + { + "name": "competitor_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Competitor Id" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 50, + "title": "Limit" + } + }, + { + "name": "since_hours", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Since Hours" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Intel Snapshots V1 Intel Snapshots Competitor Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/intel/analyze": { + "post": { + "tags": [ + "Intelligence", + "Intelligence" + ], + "summary": "Analyze competitor field for anomalies", + "description": "Analyze a specific field across a competitor's snapshots for anomalies.\n\nReturns statistics, z-score analysis, and anomaly detection.", + "operationId": "analyze_field_v1_intel_analyze_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_analyze_field_v1_intel_analyze_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Analyze Field V1 Intel Analyze Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/intel/report": { + "post": { + "tags": [ + "Intelligence", + "Intelligence" + ], + "summary": "Generate a competitive intelligence report", + "description": "Generate a competitive intelligence report for tracked competitors.\n\nAnalyzes changes over the specified period and returns a structured report\nwith a natural-language summary.", + "operationId": "generate_report_v1_intel_report_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_generate_report_v1_intel_report_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Generate Report V1 Intel Report Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/job/{job_id}": { + "get": { + "tags": [ + "Jobs", + "Jobs" + ], + "summary": "Get async job status and result", + "description": "Get async job status and result.", + "operationId": "get_job_v1_job__job_id__get", + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Job Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Job V1 Job Job Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/jobs": { + "get": { + "tags": [ + "Jobs", + "Jobs" + ], + "summary": "List jobs in a Pryfile", + "operationId": "list_jobs_v1_jobs_get", + "parameters": [ + { + "name": "path", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "pry.yml", + "title": "Path" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response List Jobs V1 Jobs Get" } } } @@ -7631,6 +4523,7 @@ "/v1/actors/create": { "post": { "tags": [ + "Marketplace", "Marketplace" ], "summary": "Create a new actor", @@ -7675,6 +4568,7 @@ "/v1/actors": { "get": { "tags": [ + "Marketplace", "Marketplace" ], "summary": "List actors in the marketplace", @@ -7731,6 +4625,7 @@ "/v1/actors/{actor_id}/run": { "post": { "tags": [ + "Marketplace", "Marketplace" ], "summary": "Run an actor", @@ -7791,9 +4686,3103 @@ } } }, + "/v1/monitor": { + "post": { + "tags": [ + "Monitoring", + "Monitoring" + ], + "summary": "Create a scheduled content monitor", + "description": "Create a scheduled monitor that tracks content changes.\n\nArgs:\n name: Human-readable monitor name\n url: Target URL to monitor\n schedule_cron: Cron expression (default: every 6 hours)\n goal: Natural language goal for meaningful-change detection\n webhook: URL to notify on meaningful changes\n use_llm: Use LLM for change judging (slower but smarter)", + "operationId": "create_monitor_endpoint_v1_monitor_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_create_monitor_endpoint_v1_monitor_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Create Monitor Endpoint V1 Monitor Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/monitor/run": { + "post": { + "tags": [ + "Monitoring", + "Monitoring" + ], + "summary": "Run a single monitor check", + "description": "Execute a monitor check immediately (outside of schedule).", + "operationId": "run_monitor_endpoint_v1_monitor_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Monitor Id" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Run Monitor Endpoint V1 Monitor Run Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/monitors": { + "get": { + "tags": [ + "Monitoring", + "Monitoring" + ], + "summary": "List all monitors", + "description": "List all registered monitors.", + "operationId": "list_monitors_endpoint_v1_monitors_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Monitors Endpoint V1 Monitors Get" + } + } + } + } + } + } + }, + "/v1/monitor/{monitor_id}": { + "delete": { + "tags": [ + "Monitoring", + "Monitoring" + ], + "summary": "Delete a monitor", + "description": "Delete a monitor and its snapshots.", + "operationId": "delete_monitor_endpoint_v1_monitor__monitor_id__delete", + "parameters": [ + { + "name": "monitor_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Monitor Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Delete Monitor Endpoint V1 Monitor Monitor Id Delete" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/monitor/check": { + "post": { + "tags": [ + "Monitoring", + "Monitoring" + ], + "summary": "Run all due monitors", + "description": "Check all monitors and run any that are due based on their cron schedule.", + "operationId": "run_due_monitors_v1_monitor_check_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Run Due Monitors V1 Monitor Check Post" + } + } + } + } + } + } + }, + "/v1/parse": { + "post": { + "tags": [ + "Parsing", + "Parsing" + ], + "summary": "Parse a document (PDF, DOCX, image, CSV, JSON)", + "description": "Parse a document (PDF, DOCX, image, CSV, JSON) to text.", + "operationId": "parse_document_v1_parse_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ParseRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Parse Document V1 Parse Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/markdown": { + "post": { + "tags": [ + "Parsing", + "Parsing" + ], + "summary": "Generate markdown with content filtering strategies", + "description": "Generate markdown with configurable content filtering.\n\nModes:\n- raw: Unfiltered markdown\n- fit: Prune boilerplate (nav, ads, footers)\n- bm25: Filter by BM25 relevance to query (requires query param)", + "operationId": "generate_markdown_v1_markdown_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_generate_markdown_v1_markdown_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Generate Markdown V1 Markdown Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/shadow-dom": { + "post": { + "tags": [ + "Parsing", + "Parsing" + ], + "summary": "Extract content from Shadow DOM", + "description": "Scrape a page and extract content from Shadow DOM components.\n\nUseful for modern web apps built with Lit, web components, or\nframeworks that use Shadow DOM encapsulation.", + "operationId": "extract_shadow_dom_v1_shadow_dom_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_extract_shadow_dom_v1_shadow_dom_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Extract Shadow Dom V1 Shadow Dom Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipeline/hook": { + "post": { + "tags": [ + "Pipeline", + "Pipeline" + ], + "summary": "Register a hook function", + "description": "Register a JavaScript hook function at a pipeline hook point.\n\nHook points: before_scrape, after_response, before_parse, after_parse,\n before_extract, after_extract, before_return, on_error", + "operationId": "register_hook_v1_pipeline_hook_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_register_hook_v1_pipeline_hook_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Register Hook V1 Pipeline Hook Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipeline/hooks": { + "get": { + "tags": [ + "Pipeline", + "Pipeline" + ], + "summary": "List all registered hooks", + "description": "List all registered hooks in the pipeline.", + "operationId": "list_pipeline_hooks_v1_pipeline_hooks_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Pipeline Hooks V1 Pipeline Hooks Get" + } + } + } + } + } + } + }, + "/v1/pipeline/run": { + "post": { + "tags": [ + "Pipeline", + "Pipeline" + ], + "summary": "Run pipeline hooks for testing", + "description": "Run hooks at a given pipeline point for testing.", + "operationId": "run_pipeline_test_v1_pipeline_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_run_pipeline_test_v1_pipeline_run_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Run Pipeline Test V1 Pipeline Run Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipelines/steps": { + "get": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "List all available pipeline step types", + "description": "List all available step types for the visual pipeline builder.\n\nEach step type includes: name, icon, description, required inputs with types,\nand expected outputs. A UI renders these as drag-and-drop blocks.", + "operationId": "list_step_types_v1_pipelines_steps_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Step Types V1 Pipelines Steps Get" + } + } + } + } + } + } + }, + "/v1/pipelines/validate": { + "post": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "Validate a pipeline definition", + "description": "Validate a pipeline definition for correctness.", + "operationId": "validate_pipeline_endpoint_v1_pipelines_validate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Pipeline" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Validate Pipeline Endpoint V1 Pipelines Validate Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipelines/run": { + "post": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "Execute a pipeline", + "description": "Execute a pipeline definition.\n\nSteps run sequentially. Each step's output is available as\n{{step_id.output_key}} in subsequent step templates.\n\nExample pipeline:\n{\n \"name\": \"Monitor Competitor Pricing\",\n \"steps\": [\n {\"id\": \"scrape_amazon\", \"type\": \"scrape\", \"inputs\": {\"url\": \"https://amazon.com/product\"}},\n {\"id\": \"extract\", \"type\": \"extract_css\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"schema\": {...}}},\n {\"id\": \"quality\", \"type\": \"quality_check\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"data\": \"{{extract.items}}\"}},\n {\"id\": \"notify\", \"type\": \"send_slack\", \"inputs\": {\"webhook_url\": \"https://hooks.slack.com/...\", \"message\": \"{{quality.quality_score}}\"}},\n ]\n}", + "operationId": "run_pipeline_endpoint_v1_pipelines_run_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_run_pipeline_endpoint_v1_pipelines_run_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Run Pipeline Endpoint V1 Pipelines Run Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipelines/save": { + "post": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "Save a pipeline definition", + "description": "Save a pipeline definition for later use.", + "operationId": "save_pipeline_endpoint_v1_pipelines_save_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Pipeline" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Save Pipeline Endpoint V1 Pipelines Save Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipelines": { + "get": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "List saved pipelines", + "description": "List all saved pipeline definitions.", + "operationId": "list_pipelines_endpoint_v1_pipelines_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Pipelines Endpoint V1 Pipelines Get" + } + } + } + } + } + } + }, + "/v1/pipelines/{pipeline_id}": { + "get": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "Get a saved pipeline", + "description": "Get a saved pipeline definition by ID.", + "operationId": "get_pipeline_endpoint_v1_pipelines__pipeline_id__get", + "parameters": [ + { + "name": "pipeline_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Pipeline Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Pipeline Endpoint V1 Pipelines Pipeline Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + }, + "delete": { + "tags": [ + "Pipelines", + "Pipelines" + ], + "summary": "Delete a saved pipeline", + "description": "Delete a saved pipeline definition.", + "operationId": "delete_pipeline_endpoint_v1_pipelines__pipeline_id__delete", + "parameters": [ + { + "name": "pipeline_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Pipeline Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Delete Pipeline Endpoint V1 Pipelines Pipeline Id Delete" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/proxy/providers": { + "get": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "List all available proxy providers", + "description": "List free + premium proxy providers with affiliate details.", + "operationId": "list_proxy_providers_v1_proxy_providers_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Proxy Providers V1 Proxy Providers Get" + } + } + } + } + } + } + }, + "/v1/proxy/signup": { + "post": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Open affiliate signup link for a provider", + "description": "Get the affiliate signup URL for a proxy provider.\n\nRecords a click for revenue tracking. The user can then sign up at that URL\nand come back to configure credentials.", + "operationId": "proxy_signup_v1_proxy_signup_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Provider" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Proxy Signup V1 Proxy Signup Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/proxy/referrals": { + "get": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "List proxy provider affiliate referrals", + "description": "Return the curated proxy provider affiliate catalog (proxy_referrals.py).\n\nThis is the marketing-friendly subset: commission, promo codes, tier\n(premium/standard/budget), and referral URLs. Useful for a \"Sign up via Pry\nand we get a cut\" UI, or for showing users premium options when their free\nproxy fails.\n\nThe full connection metadata (host, port, auth) lives in /v1/proxy/providers.", + "operationId": "list_proxy_referrals_v1_proxy_referrals_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Proxy Referrals V1 Proxy Referrals Get" + } + } + } + } + } + } + }, + "/v1/proxy/referrals/{tag}": { + "get": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Get a single proxy provider referral", + "description": "Return the affiliate info for a single proxy provider by tag.", + "operationId": "get_proxy_referral_v1_proxy_referrals__tag__get", + "parameters": [ + { + "name": "tag", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Tag" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Proxy Referral V1 Proxy Referrals Tag Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/proxy/configure": { + "post": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Configure proxy credentials", + "description": "Configure credentials for a premium proxy provider.\n\nAfter signing up via /v1/proxy/signup, the user provides their credentials here.", + "operationId": "proxy_configure_v1_proxy_configure_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_proxy_configure_v1_proxy_configure_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Proxy Configure V1 Proxy Configure Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/proxy/test": { + "get": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Test the active proxy", + "description": "Test the currently configured proxy and return its public IP.", + "operationId": "proxy_test_v1_proxy_test_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Proxy Test V1 Proxy Test Get" + } + } + } + } + } + } + }, + "/v1/proxy/status": { + "get": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Get current proxy status", + "description": "Get current proxy configuration and available providers.", + "operationId": "proxy_status_v1_proxy_status_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Proxy Status V1 Proxy Status Get" + } + } + } + } + } + } + }, + "/v1/proxy/recommend": { + "post": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Get proxy recommendation after a block", + "description": "After a scrape fails with anti-bot detection, get a recommendation\nfor which premium proxy provider to sign up with.", + "operationId": "proxy_recommend_v1_proxy_recommend_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Last Error", + "default": "" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Proxy Recommend V1 Proxy Recommend Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/proxy/clicks": { + "get": { + "tags": [ + "Proxy", + "Proxy" + ], + "summary": "Get recent proxy referral clicks", + "description": "Get recent proxy referral clicks for revenue tracking.", + "operationId": "proxy_clicks_v1_proxy_clicks_get", + "parameters": [ + { + "name": "days_back", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 30, + "title": "Days Back" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Proxy Clicks V1 Proxy Clicks Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/quality/check": { + "post": { + "tags": [ + "Quality", + "Quality" + ], + "summary": "Run data quality check on extraction results", + "description": "Run a full data quality check on extraction results.\n\nMetrics:\n- Completeness: what % of expected fields have values\n- Schema adherence: field types match expectations\n- Freshness: how old is the data\n- Anomaly detection: what changed since last extraction\n\nUse this to validate data BEFORE sending to downstream systems.", + "operationId": "quality_check_v1_quality_check_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_quality_check_v1_quality_check_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Quality Check V1 Quality Check Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/quality/stats": { + "get": { + "tags": [ + "Quality", + "Quality" + ], + "summary": "Get quality statistics for all checked URLs", + "description": "Get aggregate quality statistics across all checked URLs.", + "operationId": "quality_stats_v1_quality_stats_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Quality Stats V1 Quality Stats Get" + } + } + } + } + } + } + }, + "/v1/reconcile": { + "post": { + "tags": [ + "Reconciliation", + "Reconciliation" + ], + "summary": "Reconcile records from multiple sources into unified entities", + "description": "Reconcile records from multiple sources into matched entities.\n\nMatches records across sources using identity field similarity,\nnormalizes to a unified vertical schema, and returns entity groups\nwith confidence scores.\n\nVerticals: product, job, real_estate, review", + "operationId": "reconcile_endpoint_v1_reconcile_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_reconcile_endpoint_v1_reconcile_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Reconcile Endpoint V1 Reconcile Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/reconcile/schemas": { + "get": { + "tags": [ + "Reconciliation", + "Reconciliation" + ], + "summary": "List supported reconciliation schemas", + "description": "List all supported vertical schemas for entity reconciliation.", + "operationId": "list_schemas_v1_reconcile_schemas_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Schemas V1 Reconcile Schemas Get" + } + } + } + } + } + } + }, + "/v1/record/start": { + "post": { + "tags": [ + "Recorder", + "Recorder" + ], + "summary": "Start recording browser actions", + "operationId": "start_recording_v1_record_start_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Session Id" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Start Recording V1 Record Start Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/record/step": { + "post": { + "tags": [ + "Recorder", + "Recorder" + ], + "summary": "Record a browser action step", + "operationId": "record_step_v1_record_step_post", + "parameters": [ + { + "name": "selector", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "", + "title": "Selector" + } + }, + { + "name": "value", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "", + "title": "Value" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_record_step_v1_record_step_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Record Step V1 Record Step Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/record/export": { + "post": { + "tags": [ + "Recorder", + "Recorder" + ], + "summary": "Export recording as script", + "operationId": "export_recording_v1_record_export_post", + "parameters": [ + { + "name": "fmt", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "json", + "title": "Fmt" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Session Id" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Export Recording V1 Record Export Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/record/clear": { + "post": { + "tags": [ + "Recorder", + "Recorder" + ], + "summary": "Clear recorded actions", + "operationId": "clear_recording_v1_record_clear_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Session Id" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Clear Recording V1 Record Clear Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/referrals/catalog": { + "get": { + "tags": [ + "Referrals", + "Referrals" + ], + "summary": "Get all available referral/affiliate programs", + "description": "List all referral programs Pry supports.\n\n60+ providers across categories: LLM, hosting, domains, CDN, email,\nmonitoring, proxies, voice, media, devtools, search, CAPTCHA.", + "operationId": "get_referral_catalog_v1_referrals_catalog_get", + "parameters": [ + { + "name": "category", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "", + "title": "Category" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Referral Catalog V1 Referrals Catalog Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/referrals/stats": { + "get": { + "tags": [ + "Referrals", + "Referrals" + ], + "summary": "Get referral click and conversion stats", + "description": "Get referral tracking statistics for the last N days.", + "operationId": "get_referral_stats_v1_referrals_stats_get", + "parameters": [ + { + "name": "days_back", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "default": 30, + "title": "Days Back" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Referral Stats V1 Referrals Stats Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/referrals/click": { + "post": { + "tags": [ + "Referrals", + "Referrals" + ], + "summary": "Record a referral link click", + "description": "Record when a user clicks a referral link. Returns tracking ID.", + "operationId": "record_referral_click_v1_referrals_click_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_record_referral_click_v1_referrals_click_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Record Referral Click V1 Referrals Click Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/referrals/convert": { + "post": { + "tags": [ + "Referrals", + "Referrals" + ], + "summary": "Record a referral conversion", + "description": "Record that a referral click resulted in a conversion.", + "operationId": "record_referral_conversion_v1_referrals_convert_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_record_referral_conversion_v1_referrals_convert_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Record Referral Conversion V1 Referrals Convert Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/reports/generate": { + "post": { + "tags": [ + "Reports", + "Reports" + ], + "summary": "Generate a white-label report from scraped data", + "description": "Generate a white-label report from scraped data.\n\nReport types:\n- competitive_analysis: Competitor pricing and activity overview\n- price_monitor: Product price change tracking with visual indicators\n- seo_audit: SEO element analysis with change detection\n- content_tracker: Content change monitoring across pages\n\nBranding (optional): {\"agency_name\": \"...\", \"brand_color\": \"#hex\", \"logo_url\": \"...\"}", + "operationId": "generate_report_endpoint_v1_reports_generate_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_generate_report_endpoint_v1_reports_generate_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Generate Report Endpoint V1 Reports Generate Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/reports": { + "get": { + "tags": [ + "Reports", + "Reports" + ], + "summary": "List generated reports", + "description": "List all previously generated reports.", + "operationId": "list_reports_endpoint_v1_reports_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Reports Endpoint V1 Reports Get" + } + } + } + } + } + } + }, + "/v1/report/{report_id}": { + "get": { + "tags": [ + "Reports", + "Reports" + ], + "summary": "Get a generated report", + "description": "Get the HTML content of a generated report.", + "operationId": "get_report_v1_report__report_id__get", + "parameters": [ + { + "name": "report_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Report Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "title": "Response Get Report V1 Report Report Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/review/submit": { + "post": { + "tags": [ + "Review", + "Review" + ], + "summary": "Submit extracted data for human review", + "description": "Submit extracted data for human review before delivery.\n\nUse this when extraction confidence is low or anomalies were detected.\nData will be held in the review queue until approved or rejected.", + "operationId": "review_submit_v1_review_submit_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_review_submit_v1_review_submit_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Review Submit V1 Review Submit Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/review/{review_id}/approve": { + "post": { + "tags": [ + "Review", + "Review" + ], + "summary": "Approve a review item", + "description": "Approve a review item, allowing data to proceed to delivery.", + "operationId": "review_approve_v1_review__review_id__approve_post", + "parameters": [ + { + "name": "review_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Review Id" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_review_approve_v1_review__review_id__approve_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Review Approve V1 Review Review Id Approve Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/review/{review_id}/reject": { + "post": { + "tags": [ + "Review", + "Review" + ], + "summary": "Reject a review item", + "description": "Reject a review item, blocking data delivery.", + "operationId": "review_reject_v1_review__review_id__reject_post", + "parameters": [ + { + "name": "review_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Review Id" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_review_reject_v1_review__review_id__reject_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Review Reject V1 Review Review Id Reject Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/reviews": { + "get": { + "tags": [ + "Review", + "Review" + ], + "summary": "List reviews in the queue", + "description": "List reviews, optionally filtered by status (pending/approved/rejected).", + "operationId": "list_reviews_v1_reviews_get", + "parameters": [ + { + "name": "status", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Status" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response List Reviews V1 Reviews Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/review/{review_id}": { + "get": { + "tags": [ + "Review", + "Review" + ], + "summary": "Get review details", + "description": "Get full details of a review item including the data payload.", + "operationId": "get_review_v1_review__review_id__get", + "parameters": [ + { + "name": "review_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Review Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Get Review V1 Review Review Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/extract-with-review": { + "post": { + "tags": [ + "Review", + "Review" + ], + "summary": "Extract with automatic human review routing", + "description": "Extract data with automatic quality check and human review routing.\n\nHigh-confidence results are auto-approved.\nLow-confidence results are auto-rejected.\nMedium-confidence results go to the human review queue with Slack notification.", + "operationId": "extract_with_review_v1_extract_with_review_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_extract_with_review_v1_extract_with_review_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Extract With Review V1 Extract With Review Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/seo/analyze": { + "post": { + "tags": [ + "SEO", + "SEO" + ], + "summary": "Analyze SEO elements from a URL", + "description": "Analyze all SEO elements from a URL.\n\nReturns: title, meta description, keywords, headings (H1/H2),\ncanonical, OG tags, Twitter cards, word count, link counts,\nschema markup, hreflang tags.", + "operationId": "seo_analyze_v1_seo_analyze_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Seo Analyze V1 Seo Analyze Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/seo/track": { + "post": { + "tags": [ + "SEO", + "SEO" + ], + "summary": "Track SEO changes since last scan", + "description": "Track SEO changes since the last scan of this URL.\n\nCompares current SEO elements to previous snapshot and reports\nwhat changed (title, description, headings, etc.).", + "operationId": "seo_track_v1_seo_track_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Seo Track V1 Seo Track Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/seo/keywords": { + "post": { + "tags": [ + "SEO", + "SEO" + ], + "summary": "Analyze keyword presence in URL content", + "description": "Analyze which keywords a URL targets.\n\nChecks each keyword for:\n- Presence in title tag\n- Presence in H1 headings\n- Presence in meta description\n- Frequency in body content\n- Keyword density percentage", + "operationId": "seo_keywords_v1_seo_keywords_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_seo_keywords_v1_seo_keywords_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Seo Keywords V1 Seo Keywords Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/ultimate-scrape": { + "post": { + "tags": [ + "Scraping", + "Scraping" + ], + "summary": "Scrape with 10-tier anti-bot fallback system", + "description": "Scrape any URL using Pry's ultimate 10-tier anti-detection system.\n\nAutomatically tries: direct \u2192 cloudscraper \u2192 FlareSolverr \u2192\nundetected-chromedriver \u2192 Playwright \u2192 Googlebot \u2192 Archive.org \u2192 Google Cache\n\nReturns the first successful result with the method used.", + "operationId": "ultimate_scrape_v1_ultimate_scrape_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_ultimate_scrape_v1_ultimate_scrape_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Ultimate Scrape V1 Ultimate Scrape Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/capture/network": { + "post": { + "tags": [ + "Scraping", + "Scraping" + ], + "summary": "Extract API calls and network patterns from a page", + "description": "Extract API calls, GraphQL queries, and network patterns from a page.\n\nUseful for understanding how SPAs load data and finding hidden API endpoints.", + "operationId": "capture_network_v1_capture_network_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Capture Network V1 Capture Network Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/crawl/adaptive": { + "post": { + "tags": [ + "Scraping", + "Scraping" + ], + "summary": "Crawl with adaptive stopping based on content relevance", + "description": "Crawl a website with adaptive stopping.\n\nUses information foraging theory to decide when to stop:\n- Stops when content relevance drops below threshold\n- Stops when information gain diminishes\n- Respects max_pages and max_depth limits\n- Ideal for targeted data collection (pricing, docs, products)", + "operationId": "adaptive_crawl_v1_crawl_adaptive_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_adaptive_crawl_v1_crawl_adaptive_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Adaptive Crawl V1 Crawl Adaptive Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/session/create": { + "post": { + "tags": [ + "Sessions", + "Sessions" + ], + "summary": "Create a persistent browser session", + "description": "Create a persistent browser session.", + "operationId": "create_session_v1_session_create_post", + "parameters": [ + { + "name": "persist", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "default": true, + "title": "Persist" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Url" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Create Session V1 Session Create Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/session/destroy": { + "post": { + "tags": [ + "Sessions", + "Sessions" + ], + "summary": "Destroy a browser session with optional save", + "description": "Destroy a browser session. Optionally save state first.", + "operationId": "destroy_session_v1_session_destroy_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_destroy_session_v1_session_destroy_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Destroy Session V1 Session Destroy Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/sessions": { + "get": { + "tags": [ + "Sessions", + "Sessions" + ], + "summary": "List all persistent sessions", + "description": "List all persistent sessions (active + saved).", + "operationId": "list_sessions_v1_sessions_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response List Sessions V1 Sessions Get" + } + } + } + } + } + } + }, + "/v1/session/save": { + "post": { + "tags": [ + "Sessions", + "Sessions" + ], + "summary": "Save current session state to disk", + "description": "Save a session's current state (cookies, storage) to disk.", + "operationId": "save_session_state_v1_session_save_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Session Id" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Save Session State V1 Session Save Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/session/restore": { + "post": { + "tags": [ + "Sessions", + "Sessions" + ], + "summary": "Restore a saved session", + "description": "Restore a session from disk into a browser context.", + "operationId": "restore_session_v1_session_restore_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Session Id" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Restore Session V1 Session Restore Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/share": { + "post": { + "tags": [ + "Share", + "Share" + ], + "summary": "Share scraped content via link", + "operationId": "share_scrape_v1_share_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Data" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Share Scrape V1 Share Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/share/{share_id}": { + "get": { + "tags": [ + "Share", + "Share" + ], + "summary": "View shared content", + "operationId": "view_share_share__share_id__get", + "parameters": [ + { + "name": "share_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Share Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": {} + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v0/stats": { + "get": { + "tags": [ + "Stats", + "Stats" + ], + "summary": "Get cache, rate limiter, and session stats", + "operationId": "stats_v0_stats_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Stats V0 Stats Get" + } + } + } + } + } + } + }, + "/v1/structure/check": { + "post": { + "tags": [ + "Structure", + "Structure" + ], + "summary": "Check if CSS selectors still match on a page", + "description": "Check if CSS selectors still match on a page.\n\nUse this to verify your scraper templates still work after\na site redesign. Returns per-selector match status.\n\nSelectors format: [{\"name\": \"title\", \"selector\": \"h1\", \"type\": \"css\"}]", + "operationId": "structure_check_v1_structure_check_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_structure_check_v1_structure_check_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Structure Check V1 Structure Check Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/structure/monitor": { + "post": { + "tags": [ + "Structure", + "Structure" + ], + "summary": "Monitor page structure changes over time", + "description": "Monitor page structure changes and detect broken selectors.\n\nCompares current selector match status to previous checks.\nAlerts when selectors stop matching (site redesign detected).\n\nUse this as a pre-scrape health check for your templates.", + "operationId": "structure_monitor_v1_structure_monitor_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_structure_monitor_v1_structure_monitor_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Structure Monitor V1 Structure Monitor Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/structure/check-template": { + "post": { + "tags": [ + "Structure", + "Structure" + ], + "summary": "Verify a scraper template still works", + "description": "Check if a pre-built scraper template still works against a URL.\n\nExtracts the template's selectors and checks each one.\nIf selectors fail, the template needs updating.", + "operationId": "structure_check_template_v1_structure_check_template_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_structure_check_template_v1_structure_check_template_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Structure Check Template V1 Structure Check Template Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/training/classify-license": { + "post": { + "tags": [ + "Training", + "Training" + ], + "summary": "Classify content license for AI training", + "description": "Classify the license of scraped content for AI training compliance.\n\nReturns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use),\ntier (permissive/copyleft/restrictive/conditional), and confidence.", + "operationId": "classify_license_endpoint_v1_training_classify_license_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string", + "title": "Text" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Classify License Endpoint V1 Training Classify License Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/training/clean": { + "post": { + "tags": [ + "Training", + "Training" + ], + "summary": "Strip PII and copyright from content", + "description": "Strip PII and copyright content for AI training clean room.\n\nRemoves emails, phones, SSNs, credit cards, IPs, and (optionally) names.\nAlso strips copyright notices and near-verbatim copyright content.", + "operationId": "clean_content_v1_training_clean_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_clean_content_v1_training_clean_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Clean Content V1 Training Clean Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/training/export": { + "post": { + "tags": [ + "Training", + "Training" + ], + "summary": "Export a clean AI training dataset", + "description": "Export scraped content as a clean AI training dataset.\n\nEach record should have: content, url, and optional metadata.\n\nFeatures:\n- Per-record provenance tracking (source URL, timestamp, extraction method)\n- PII stripping (email, phone, SSN, CC, IP)\n- Copyright verbatim text removal\n- License classification\n- Compliance report generation", + "operationId": "export_dataset_v1_training_export_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_export_dataset_v1_training_export_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Export Dataset V1 Training Export Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/training/compliance/{dataset_id}": { + "get": { + "tags": [ + "Training", + "Training" + ], + "summary": "Generate compliance report for a dataset", + "description": "Generate a compliance report for an exported training dataset.\n\nReport includes: data provenance, PII/copyright removal stats,\nlicense classification, and legal recommendations.", + "operationId": "compliance_report_v1_training_compliance__dataset_id__get", + "parameters": [ + { + "name": "dataset_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "title": "Dataset Id" + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Compliance Report V1 Training Compliance Dataset Id Get" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/transform": { + "post": { + "tags": [ + "Transform", + "Transform" + ], + "summary": "Transform data to multiple formats", + "operationId": "transform_data_v1_transform_post", + "parameters": [ + { + "name": "format", + "in": "query", + "required": false, + "schema": { + "type": "string", + "default": "csv", + "title": "Format" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + }, + "title": "Data" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true, + "title": "Response Transform Data V1 Transform Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/pipe": { + "post": { + "tags": [ + "Transform", + "Transform" + ], + "summary": "Scrape and transform via data pipeline", + "operationId": "data_pipeline_v1_pipe_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Data" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Data Pipeline V1 Pipe Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/vision": { + "post": { + "tags": [ + "Vision", + "Vision" + ], + "summary": "Analyze an image with a free vision model", + "description": "Analyze an image with a free vision model.\n\nProvide ONE of: image (base64 or data URI), url (auto-screenshot),\nor file_path (local PNG/JPG).\n\nAuto-falls-back across 5 free OpenRouter vision models if the\nrequested one is rate-limited.", + "operationId": "vision_v1_vision_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_vision_v1_vision_post" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Vision V1 Vision Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/vision/models": { + "get": { + "tags": [ + "Vision", + "Vision" + ], + "summary": "List free vision-capable models on OpenRouter", + "description": "List free vision-capable models on OpenRouter.", + "operationId": "vision_models_v1_vision_models_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response Vision Models V1 Vision Models Get" + } + } + } + } + } + } + }, "/v1/webhooks/test": { "post": { "tags": [ + "Webhooks", "Webhooks" ], "summary": "Test webhook delivery", @@ -7838,6 +7827,7 @@ "/v1/webhooks/dead-letter": { "get": { "tags": [ + "Webhooks", "Webhooks" ], "summary": "Get failed webhook deliveries", @@ -7858,6 +7848,305 @@ } } } + }, + "/v1/x402/pricing": { + "get": { + "tags": [ + "x402", + "x402" + ], + "summary": "Get x402 pricing for all paid operations", + "description": "Get the price list for pay-per-scrape operations.", + "operationId": "x402_pricing_v1_x402_pricing_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Pricing V1 X402 Pricing Get" + } + } + } + } + } + } + }, + "/v1/x402/payment": { + "post": { + "tags": [ + "x402", + "x402" + ], + "summary": "Create a x402 payment request", + "description": "Create a x402 payment request for a paid operation.\n\nReturns payment details (wallet, amount, asset) for the client to pay.", + "operationId": "x402_payment_request_v1_x402_payment_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_x402_payment_request_v1_x402_payment_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Payment Request V1 X402 Payment Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/x402/verify": { + "post": { + "tags": [ + "x402", + "x402" + ], + "summary": "Verify a x402 payment was settled", + "description": "Verify a x402 payment has been settled on-chain via facilitator router.", + "operationId": "x402_verify_payment_v1_x402_verify_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_x402_verify_payment_v1_x402_verify_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Verify Payment V1 X402 Verify Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/x402/require-payment": { + "post": { + "tags": [ + "x402", + "x402" + ], + "summary": "Generate a 402 Payment Required response", + "description": "Generate a 402 Payment Required response for a paid endpoint.", + "operationId": "x402_require_payment_v1_x402_require_payment_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_x402_require_payment_v1_x402_require_payment_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Require Payment V1 X402 Require Payment Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/x402/batch-payment": { + "post": { + "tags": [ + "x402", + "x402" + ], + "summary": "Create a batch x402 payment", + "description": "Create a single x402 payment covering multiple operations.\n\nReturns a PaymentRequired body with the combined amount. After paying,\nsubmit the tx to POST /v1/x402/batch-verify.", + "operationId": "x402_batch_payment_v1_x402_batch_payment_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Payload" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Batch Payment V1 X402 Batch Payment Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/x402/batch-verify": { + "post": { + "tags": [ + "x402", + "x402" + ], + "summary": "Verify a batch x402 payment", + "description": "Verify the on-chain payment for a batch and mark it paid.", + "operationId": "x402_batch_verify_v1_x402_batch_verify_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Payload" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Batch Verify V1 X402 Batch Verify Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, + "/v1/x402/pay": { + "post": { + "tags": [ + "x402", + "x402" + ], + "summary": "Process x402 payment and get access token", + "description": "Process an x402 payment and return an access token.\n\nFlow:\n1. User gets 402 from a paid endpoint\n2. User sends USDC to the wallet in the 402 response\n3. User calls this endpoint with the tx_hash\n4. Pry verifies the transaction through the facilitator router\n5. Returns access token (payment_id) to use in X-Payment-Hash header", + "operationId": "x402_pay_v1_x402_pay_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Body_x402_pay_v1_x402_pay_post" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object", + "title": "Response X402 Pay V1 X402 Pay Post" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } } }, "components": { @@ -8030,6 +8319,37 @@ ], "title": "BatchFileRequest" }, + "BatchItem": { + "properties": { + "template_id": { + "type": "string", + "title": "Template Id" + }, + "url": { + "type": "string", + "title": "Url" + }, + "options": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Options" + } + }, + "type": "object", + "required": [ + "template_id", + "url" + ], + "title": "BatchItem", + "description": "A single template execution request within a batch." + }, "Body_adaptive_crawl_v1_crawl_adaptive_post": { "properties": { "url": { @@ -10356,4 +10676,4 @@ } } } -} +} \ No newline at end of file diff --git a/routers/__init__.py b/routers/__init__.py index c8f45ce..79ca4fd 100644 --- a/routers/__init__.py +++ b/routers/__init__.py @@ -43,7 +43,56 @@ Licensed under MIT. See LICENSE. # Licensed under MIT. See LICENSE. __all__: list[str] = [ - "auth_router", - "health_router", - "templates_router", + "advanced_router", + "agency_router", + "ai_router", + "alerts_router", + "analysis_router", + "auth_api_router", + "automation_router", + "batch_router", + "circuit_breaker_router", + "commerce_router", + "compliance_router", + "config_router", + "costing_router", + "crm_router", + "dashboard_router", + "email_router", + "enrichment_router", + "execute_router", + "export_router", + "extraction_router", + "freshness_router", + "gdpr_router", + "health_api_router", + "integrations_router", + "intelligence_router", + "jobs_router", + "marketplace_router", + "monitoring_router", + "parsing_router", + "pipeline_router", + "pipelines_router", + "proxy_router", + "quality_router", + "reconciliation_router", + "recorder_router", + "referrals_router", + "reports_router", + "review_router", + "scraping_api_router", + "seo_router", + "sessions_router", + "share_router", + "stats_router", + "structure_router", + "system_router", + "templates_api_router", + "training_router", + "transform_router", + "untagged_router", + "vision_router", + "webhooks_router", + "x402_router", ] diff --git a/routers/advanced.py b/routers/advanced.py new file mode 100644 index 0000000..76d2675 --- /dev/null +++ b/routers/advanced.py @@ -0,0 +1,329 @@ +"""Pry โ€” Advanced router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import base64 +import logging +from typing import Any + +import httpx +from fastapi import APIRouter, Body + +from client import get_client + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Advanced"]) + +@router.post( + "/v1/tls/impersonate", + tags=["Advanced"], + summary="Fetch a URL with TLS fingerprint impersonation", +) +async def tls_impersonate( + url: str = Body(...), + impersonate: str = Body("chrome131"), + proxy: str = Body(""), +) -> dict[str, Any]: + """Fetch a URL while impersonating a real browser's TLS fingerprint. + Bypasses JA3/JA4 fingerprinting that blocks 80%+ of bot traffic.""" + from tls_fingerprint import TLSScraper + + s = TLSScraper() + if not s.is_available(): + return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"} + result = await s.fetch(url, impersonate=impersonate, proxy=proxy) + return result + + +@router.post( + "/v1/tls/rotate", + tags=["Advanced"], + summary="Try multiple browser fingerprints until one succeeds", +) +async def tls_rotate( + url: str = Body(...), + proxy: str = Body(""), +) -> dict[str, Any]: + """Try multiple browser fingerprints until one succeeds (anti-fingerprint rotation).""" + from tls_fingerprint import TLSScraper + + s = TLSScraper() + if not s.is_available(): + return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"} + result = await s.fetch_with_rotation(url, proxy=proxy) + return result + + +@router.post( + "/v1/camoufox/fetch", tags=["Advanced"], summary="Fetch with Camoufox anti-detection Firefox" +) +async def camoufox_fetch( + url: str = Body(...), + profile: str = Body("chrome_windows"), + wait_selector: str = Body(""), + proxy: str = Body(""), +) -> dict[str, Any]: + """Fetch a URL using Camoufox (Firefox anti-detection browser). + + Camoufox patches Firefox at the source level for maximum stealth. + This is more effective than Playwright for sites with advanced + fingerprinting (DataDome, PerimeterX, advanced Cloudflare). + """ + from camoufox_integration import CamoufoxBrowser + + b = CamoufoxBrowser() + if not b.is_available(): + return { + "success": False, + "error": "camoufox not installed. Run: pip install camoufox && python -m camoufox fetch", + } + result = await b.fetch(url, profile=profile, wait_selector=wait_selector, proxy=proxy) + return result + + +@router.post( + "/v1/graphql/discover", tags=["Advanced"], summary="Discover GraphQL endpoints for a site" +) +async def graphql_discover(base_url: str = Body(...)) -> dict[str, Any]: + """Auto-discover GraphQL endpoints for a website.""" + from graphql_discovery import GraphQLDiscovery + + g = GraphQLDiscovery() + found = await g.discover(base_url) + return {"success": True, "data": {"found": found, "count": len(found)}} + + +@router.post( + "/v1/graphql/introspect", tags=["Advanced"], summary="Run GraphQL introspection on an endpoint" +) +async def graphql_introspect(endpoint: str = Body(...)) -> dict[str, Any]: + """Run GraphQL introspection query against a discovered endpoint.""" + from graphql_discovery import GraphQLDiscovery + + g = GraphQLDiscovery() + result = await g.introspect(endpoint) + return {"success": "error" not in result, "data": result} + + +@router.post("/v1/graphql/query", tags=["Advanced"], summary="Execute a GraphQL query") +async def graphql_query( + endpoint: str = Body(...), + query: str = Body(...), + variables: dict[str, Any] | None = Body(None), +) -> dict[str, Any]: + """Execute a GraphQL query against a discovered endpoint.""" + from graphql_discovery import GraphQLDiscovery + + g = GraphQLDiscovery() + result = await g.query(endpoint, query, variables) + return {"success": "error" not in result, "data": result} + + +@router.post( + "/v1/schema/extract", + tags=["Advanced"], + summary="Extract Schema.org structured data from a page", +) +async def schema_extract(url: str = Body(...)) -> dict[str, Any]: + """Extract Schema.org/JSON-LD/Microdata/RDFa structured data from a URL. + Most modern sites embed structured data โ€” extract it directly instead of + scraping HTML. 100x faster and more accurate.""" + from schema_extraction import SchemaExtractor + + client = await get_client() + resp = await client.get(url, timeout=30) + e = SchemaExtractor() + result = e.extract_all(resp.text) + return {"success": True, "data": result} + + +@router.post( + "/v1/schema/extract-html", + tags=["Advanced"], + summary="Extract Schema.org structured data from raw HTML", +) +async def schema_extract_html(html: str = Body(...)) -> dict[str, Any]: + """Extract Schema.org/JSON-LD/Microdata/RDFa from a raw HTML string. + Useful when you've already fetched the page and want to extract structured data.""" + from schema_extraction import SchemaExtractor + + e = SchemaExtractor() + result = e.extract_all(html) + return {"success": True, "data": result} + + +@router.post( + "/v1/ws/scrape", + tags=["Advanced"], + summary="Scrape data from a WebSocket endpoint", +) +async def ws_scrape( + url: str = Body(...), + max_messages: int = Body(100), + timeout: int = Body(30), + message_filter: str = Body(""), +) -> dict[str, Any]: + """Connect to a WebSocket and capture the data stream. + Useful for SPAs and real-time apps that load data via WebSocket.""" + from websocket_scraper import WebSocketScraper + + s = WebSocketScraper(max_messages=max_messages, timeout=timeout) + result = await s.scrape_websocket(url, message_filter=message_filter) + return {"success": result.get("success", False), "data": result} + + +@router.post( + "/v1/sse/scrape", + tags=["Advanced"], + summary="Scrape data from a Server-Sent Events endpoint", +) +async def sse_scrape( + url: str = Body(...), + max_events: int = Body(50), + timeout: int = Body(30), + event_filter: str = Body(""), +) -> dict[str, Any]: + """Connect to a Server-Sent Events endpoint and capture events.""" + from websocket_scraper import WebSocketScraper + + s = WebSocketScraper(timeout=timeout) + result = await s.scrape_sse(url, event_filter=event_filter, max_events=max_events) + return {"success": result.get("success", False), "data": result} + + +@router.post( + "/v1/cookies/warm", + tags=["Advanced"], + summary="Warm cookies for a domain by browsing legitimate pages", +) +async def warm_cookies( + target_domain: str = Body(...), + pages_to_visit: int = Body(3), +) -> dict[str, Any]: + """Pre-age cookies for a domain to bypass anti-bot detection. + Visits legitimate pages first to build realistic browsing history.""" + from cookie_warmer import CookieWarmer + + w = CookieWarmer() + result = await w.warm_for_site(target_domain, pages_to_visit=pages_to_visit) + return {"success": result.get("success", False), "data": result} + + +@router.get( + "/v1/cookies/sessions", + tags=["Advanced"], + summary="List all warmed cookie sessions", +) +async def list_cookie_sessions() -> dict[str, Any]: + from cookie_warmer import CookieWarmer + + w = CookieWarmer() + return {"success": True, "data": w.list_sessions()} + + +@router.post( + "/v1/pdf/extract", + tags=["Advanced"], + summary="Extract tables and text from a PDF", +) +async def extract_pdf( + pdf_url: str = Body(...), + method: str = Body("pdfplumber"), +) -> dict[str, Any]: + """Download a PDF and extract structured tables and text.""" + from pdf_extractor import PDFTableExtractor + + client = await get_client() + try: + resp = await client.get(pdf_url, timeout=60) + if not resp.is_success: + return { + "success": False, + "error": f"Failed to download: HTTP {resp.status_code}", + } + e = PDFTableExtractor() + result = e.extract(resp.content, method=method) + return {"success": "error" not in result, "data": result} + except (httpx.HTTPError, httpx.RequestError) as e: + return {"success": False, "error": str(e)[:300]} + + +@router.post( + "/v1/ocr/extract", + tags=["Advanced"], + summary="Extract text from an image using Tesseract", +) +async def ocr_extract( + image_url: str = Body(""), + image_base64: str = Body(""), +) -> dict[str, Any]: + """Extract text from an image via URL or base64.""" + from ocr_extractor import ImageOCR + + o = ImageOCR() + if image_url: + result = await o.extract_from_url(image_url) + elif image_base64: + data = base64.b64decode(image_base64.split(",", 1)[-1]) + result = o.extract_from_bytes(data) + else: + return {"success": False, "error": "Provide image_url or image_base64"} + return result + + +@router.post( + "/v1/dedup/check", + tags=["Advanced"], + summary="Check if content is a near-duplicate using SimHash", +) +async def check_duplicate( + text: str = Body(...), + threshold: float = Body(0.85), +) -> dict[str, Any]: + """Check if text is a near-duplicate of recently seen content using SimHash.""" + from dedup import SimHash + + h = SimHash.hash(text) + return { + "success": True, + "data": { + "hash": h, + "text_length": len(text), + "threshold": threshold, + }, + } + + +@router.get( + "/v1/behavior/simulate", + tags=["Advanced"], + summary="Generate human-like behavior patterns for testing", +) +async def get_behavior_simulation(action: str = "mouse") -> dict[str, Any]: + """Generate realistic human behavior patterns (mouse path, scroll, typing, etc.)""" + from behavioral_biometrics import behavior + + if action == "mouse": + path = behavior.mouse_path((0, 0), (800, 600)) + elif action == "scroll": + path = behavior.scroll_pattern(3000) + elif action == "typing": + path = behavior.typing_pattern("the quick brown fox jumps over") + elif action == "click": + return { + "success": True, + "data": {"delay_ms": behavior.click_decision_delay() * 1000}, + } + else: + return { + "success": False, + "error": f"Unknown action: {action}. Use mouse|scroll|typing|click", + } + return {"success": True, "data": path} diff --git a/routers/agency.py b/routers/agency.py new file mode 100644 index 0000000..400a245 --- /dev/null +++ b/routers/agency.py @@ -0,0 +1,111 @@ +"""Pry โ€” Agency router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Agency"]) + +@router.post("/v1/agency/create", tags=["Agency"], summary="Create a white-label agency profile") +async def create_agency_endpoint( + name: str = Body(...), + owner_email: str = Body(...), + custom_domain: str = Body(""), + brand_color: str = Body("#f59e0b"), + logo_url: str = Body(""), +) -> dict[str, Any]: + """Create a white-label agency profile for reselling Pry. + + Agencies get: + - Custom branding (colors, logo, domain) + - Client management with sub-accounts + - Usage analytics and quota management + - API key management for each client + """ + from agency import create_agency + + result = create_agency(name, owner_email, custom_domain, brand_color, logo_url) + return {"success": result["success"], "data": result} + + +@router.get("/v1/agency/{agency_id}", tags=["Agency"], summary="Get agency profile") +async def get_agency_endpoint(agency_id: str) -> dict[str, Any]: + """Get agency profile details.""" + from agency import get_agency + + result = get_agency(agency_id) + if not result: + raise NotFoundError(f"Agency not found: {agency_id}") + return {"success": True, "data": result} + + +@router.put("/v1/agency/{agency_id}/branding", tags=["Agency"], summary="Update agency branding") +async def update_branding( + agency_id: str, + name: str | None = Body(None), + brand_color: str | None = Body(None), + logo_url: str | None = Body(None), + custom_domain: str | None = Body(None), +) -> dict[str, Any]: + """Update white-label branding for an agency.""" + from agency import update_agency_branding + + result = update_agency_branding(agency_id, name, brand_color, logo_url, custom_domain) + return {"success": result["success"], "data": result} + + +@router.post("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account") +async def create_client_endpoint( + agency_id: str, + client_name: str = Body(...), + client_email: str = Body(...), + monthly_quota: int = Body(10000), +) -> dict[str, Any]: + """Create a client sub-account under an agency. + + Each client gets their own API key and usage quota. + """ + from agency import create_client + + result = create_client(agency_id, client_name, client_email, monthly_quota) + return {"success": result["success"], "data": result} + + +@router.get("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="List agency clients") +async def list_clients_endpoint(agency_id: str) -> dict[str, Any]: + """List all clients under an agency.""" + from agency import list_clients + + clients = list_clients(agency_id) + return {"success": True, "data": {"clients": clients, "total": len(clients)}} + + +@router.get("/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics") +async def get_analytics(agency_id: str) -> dict[str, Any]: + """Get aggregate usage analytics for an agency.""" + from agency import get_agency_analytics + + result = get_agency_analytics(agency_id) + return {"success": True, "data": result} + + +@router.get("/v1/client/{client_id}/quota", tags=["Agency"], summary="Check client quota usage") +async def check_quota(client_id: str) -> dict[str, Any]: + """Check a client's current quota usage and remaining capacity.""" + from agency import check_client_quota + + result = check_client_quota(client_id) + return {"success": result["success"], "data": result} diff --git a/routers/ai.py b/routers/ai.py new file mode 100644 index 0000000..480ee10 --- /dev/null +++ b/routers/ai.py @@ -0,0 +1,41 @@ +"""Pry โ€” AI router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging + +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["AI"]) + +@router.get("/v1/ai/gpt-manifest", tags=["AI"], summary="Get GPT Action manifest for ChatGPT") +async def get_gpt_manifest() -> JSONResponse: + """Get the GPT Action manifest for ChatGPT integration. + + Add this to your GPT configuration in ChatGPT to give it + web scraping capabilities through Pry. + """ + from ai_plugin import get_gpt_action_manifest + + return JSONResponse(content=get_gpt_action_manifest()) + + +@router.get("/v1/ai/mcp-config", tags=["AI"], summary="Get MCP server config for Claude/Cursor") +async def get_mcp_config() -> JSONResponse: + """Get the MCP server configuration for Claude/Cursor. + + Add this to your AI tool's MCP configuration file to let + Claude and Cursor scrape the web through Pry. + """ + from ai_plugin import get_mcp_server_config + + return JSONResponse(content={"success": True, "data": get_mcp_server_config()}) diff --git a/routers/alerts.py b/routers/alerts.py new file mode 100644 index 0000000..1e5a969 --- /dev/null +++ b/routers/alerts.py @@ -0,0 +1,67 @@ +"""Pry โ€” Alerts router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Alerts"]) + +@router.post("/v1/alert/send", tags=["Alerts"], summary="Send an alert to any channel") +async def send_alert_endpoint( + channel: str = Body(...), + title: str = Body("Pry Alert"), + message: str = Body(...), + config: dict[str, Any] = Body(...), + severity: str = Body("info"), +) -> dict[str, Any]: + """Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email. + + Config varies by channel: + - slack: {"webhook_url": "..."} + - discord: {"webhook_url": "..."} + - teams: {"webhook_url": "..."} + - telegram: {"bot_token": "...", "chat_id": "..."} + - sms: {"phone": "+1234567890", "twilio_sid": "...", "twilio_token": "...", "twilio_from": "..."} + - email: {"recipient": "user@example.com", "smtp_host": "...", "smtp_user": "..."} + """ + from alerter import send_alert + + result = await send_alert(channel, title, message, config, severity) + return {"success": result["success"], "data": result} + + +@router.get("/v1/alert/channels", tags=["Alerts"], summary="List supported alert channels") +async def list_channels() -> dict[str, Any]: + """List all supported alert channels and their config requirements.""" + return { + "success": True, + "data": { + "channels": [ + {"id": "slack", "name": "Slack", "config_fields": ["webhook_url"]}, + {"id": "discord", "name": "Discord", "config_fields": ["webhook_url"]}, + {"id": "teams", "name": "Microsoft Teams", "config_fields": ["webhook_url"]}, + {"id": "telegram", "name": "Telegram", "config_fields": ["bot_token", "chat_id"]}, + { + "id": "sms", + "name": "SMS (Twilio)", + "config_fields": ["phone", "twilio_sid", "twilio_token", "twilio_from"], + }, + { + "id": "email", + "name": "Email", + "config_fields": ["recipient", "smtp_host", "smtp_user"], + }, + ] + }, + } diff --git a/routers/analysis.py b/routers/analysis.py new file mode 100644 index 0000000..ff8a00d --- /dev/null +++ b/routers/analysis.py @@ -0,0 +1,199 @@ +"""Pry โ€” Analysis router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import asyncio +import difflib +import json +import logging +import re +from typing import Any + +import httpx +from fastapi import APIRouter, Body + +from client import get_client +from deps import advanced, scraper +from errors import ScrapeError +from settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Analysis"]) + +@router.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs") +async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]: + """Scrape two URLs and compare their content. Shows additions, deletions, changes.""" + r1, r2 = await asyncio.gather(scraper.scrape(url1), scraper.scrape(url2)) + c1, c2 = r1.get("content", ""), r2.get("content", "") + diff = list( + difflib.unified_diff( + c1.splitlines(keepends=True), + c2.splitlines(keepends=True), + fromfile=url1, + tofile=url2, + n=3, + ) + ) + return { + "success": True, + "data": { + "url1": url1, + "url2": url2, + "diff": diff[:200], + "changes": len(diff), + "identical": len(diff) == 0, + }, + } + + +@router.post("/v1/watch", tags=["Analysis"], summary="Watch a page for changes") +async def watch_page( + url: str = Body(...), webhook: str = Body(...), interval: int = 3600, selector: str = "" +) -> dict[str, Any]: + """Watch a page for changes. Accepts a webhook URL for notification. + First call registers the watch, subsequent calls compare and notify. + Firecrawl charges $99/mo for this feature.""" + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError("Could not scrape URL") + content = result.get("content", "") + diff_result = await advanced.track_diff(url, content) + # Fire webhook asynchronously on first registration (webhook support) + if webhook and diff_result["status"] == "registered": + asyncio.create_task(_fire_watch_webhook(webhook, url, diff_result)) + return { + "success": True, + "data": { + "url": url, + "status": diff_result["status"], + "version": diff_result["version"], + "webhook": webhook, + "interval": interval, + "message": "Page registered for change tracking.", + }, + } + + +@router.post("/v1/summarize", tags=["Analysis"], summary="AI summarize scraped content") +async def summarize(url: str = Body(...), max_words: int = 100) -> dict[str, Any]: + """Scrape + AI summarize using local Ollama. Free, private.""" + result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error", "Scrape failed")) + summary = await advanced.summarize(result.get("content", ""), max_words) + return {"success": True, "data": {"url": url, **summary}} + + +@router.post("/v1/diff", tags=["Analysis"], summary="Track page changes over time") +async def diff(url: str = Body(...), content: str | None = Body(None)) -> dict[str, Any]: + """Track page changes over time. Returns diff from last scrape.""" + if content: + diff_result = await advanced.track_diff(url, content) + else: + result = await scraper.scrape(url) + if result.get("status") != "ok": + raise ScrapeError("Could not scrape URL") + diff_result = await advanced.track_diff(url, result.get("content", "")) + return {"success": True, "data": diff_result} + + +@router.post("/v1/categorize", tags=["Analysis"], summary="AI-categorize scraped content") +async def categorize(url: str = Body(...)) -> dict[str, Any]: + """AI-categorize scraped content into topic tags.""" + result = await scraper.scrape(url) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Categorize failed") + tags = await advanced.categorize(result.get("content", "")) + kw = advanced.keyword_density(result.get("content", "")) + readability = advanced.readability(result.get("content", "")) + return { + "success": True, + "data": {"url": url, "tags": tags, "keywords": kw[:10], "readability": readability}, + } + + +@router.post("/v1/suggest", tags=["Analysis"], summary="AI-suggest schema fields for a URL") +async def suggest_schema(data: dict[str, Any] = Body(...)) -> dict[str, Any]: + url = data.get("url", "") + + result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30}) + content = result.get("content", "") + html = "" + try: + client = await get_client() + resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) + html = resp.text + except httpx.HTTPError: + logger.warning("suggest_schema_fetch_failed", extra={"url": url}) + schema = {"_page_title": result.get("title", "")} + candidates: dict[str, str] = {} + for pattern, field in [ + (r'["\']((?:price|cost|amount)[^"\']*)["\']', "price"), + (r'["\'](product-title|product-name|item-name)[^"\']*["\']', "name"), + (r'["\'](?:product-image|main-image|featured-image)[^"\']*["\']', "image"), + (r'["\'](?:description|product-desc|item-desc)[^"\']*["\']', "description"), + (r'["\'](?:rating|stars|review-score)[^"\']*["\']', "rating"), + (r'["\'](?:stock|availability|status)[^"\']*["\']', "stock"), + ]: + found = re.findall(pattern, html.lower()) + if found: + candidates[field] = f'[class*="{found[0][:15]}"]' + if "name" not in candidates: + h1 = re.findall(r']*class=["\']([^"\']*)["\']', html) + if h1: + candidates["name"] = f"h1.{h1[0].replace(' ', '.')}" + schema["suggested"] = candidates + if content and len(content) > 200: + try: + prompt = ( + "Analyze this page. Return ONLY JSON with field names as keys and CSS selectors as values. Look for: price, name, description, image, rating, stock.\n\n" + + content[:4000] + ) + client = await get_client() + ollama_url = settings.ollama_url + resp = await client.post( + f"{ollama_url}/api/generate", + json={ + "model": "qwen2.5-coder:3b", + "prompt": prompt, + "stream": False, + "options": {"num_ctx": 8192, "temperature": 0.1}, + }, + timeout=30, + ) + if resp.status_code == 200: + llm_raw = resp.json().get("response", "") + llm_match = re.search(r"\{.*\}", llm_raw, re.S) + if llm_match: + try: + llm = json.loads(llm_match.group(0)) + if isinstance(llm, dict): + schema["llm_suggested"] = llm + except json.JSONDecodeError: + logger.warning("llm_schema_parse_failed") + except httpx.HTTPError: + logger.warning("ollama_suggest_schema_failed") + return {"success": True, "data": schema} + + +async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any]) -> None: + """Fire a webhook notification for watch events.""" + try: + client = await get_client() + await client.post( + webhook, + json={"event": "watch_update", "url": url, "data": diff_result}, + timeout=10, + ) + except (httpx.HTTPError, httpx.RequestError): + logger.exception("watch_webhook_failed", extra={"url": url, "webhook": webhook}) + + +logger = logging.getLogger(__name__) diff --git a/routers/automation.py b/routers/automation.py new file mode 100644 index 0000000..8cceaf8 --- /dev/null +++ b/routers/automation.py @@ -0,0 +1,77 @@ +"""Pry โ€” Automation router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +import pydantic +from fastapi import APIRouter, Body +from pydantic import BaseModel + +from deps import automator +from errors import ExternalServiceError, PryError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Automation"]) + +@router.post("/v1/automate", tags=["Automation"], summary="Execute browser automation steps") +async def automate(request: AutomateRequest) -> dict[str, Any]: + """Execute browser automation steps. Sessions persist for login flows.""" + try: + steps = [s.model_dump() for s in request.steps] + result = await automator.run_steps( + steps=steps, + session_id=request.session_id, + headless=True if request.headless is None else request.headless, + viewport=request.viewport, + ) + return {"success": True, "data": result} + except PryError: + raise + except pydantic.ValidationError as e: + raise ExternalServiceError(str(e)) from e + + +@router.post("/v1/screenshot", tags=["Automation"], summary="Take a screenshot of a URL") +async def screenshot( + url: str = Body(..., embed=True), session_id: str | None = None +) -> dict[str, Any]: + """Take a screenshot of a URL. Returns base64 PNG.""" + try: + result = await automator.run_steps( + steps=[{"action": "navigate", "url": url}, {"action": "screenshot"}], + session_id=session_id, + ) + screenshot_data = None + for step in result.get("steps", []): + if step.get("action") == "screenshot": + screenshot_data = step.get("screenshot") + return {"success": True, "data": {"screenshot": screenshot_data or ""}} + except PryError: + raise + except pydantic.ValidationError as e: + raise ExternalServiceError(str(e)) from e + + +class AutomateRequest(BaseModel): + session_id: str | None = None + steps: list[AutomateStep] + headless: bool | None = True + viewport: dict[str, int] | None = None + + +class AutomateStep(BaseModel): + action: str + selector: str | None = None + value: str | None = None + url: str | None = None + timeout: int | None = 30000 + wait_until: str | None = "networkidle" diff --git a/routers/batch.py b/routers/batch.py new file mode 100644 index 0000000..aba2b53 --- /dev/null +++ b/routers/batch.py @@ -0,0 +1,36 @@ +"""Pry โ€” Batch router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter +from pydantic import BaseModel + +from pryextras import BatchProcessor + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Batch"]) + +@router.post("/v1/batch-file", tags=["Batch"], summary="Batch scrape URLs from a file with templates") +async def batch_from_file(request: BatchFileRequest) -> dict[str, Any]: + bp = BatchProcessor() + results = await bp.from_file( + request.filepath, request.template, request.timeout or 30, request.max_urls or 1000 + ) + return {"success": True, "data": {"total": len(results), "results": results}} + + +class BatchFileRequest(BaseModel): + filepath: str + template: dict[str, str] + timeout: int | None = 30 + max_urls: int | None = 1000 diff --git a/routers/circuit_breaker.py b/routers/circuit_breaker.py new file mode 100644 index 0000000..1076c35 --- /dev/null +++ b/routers/circuit_breaker.py @@ -0,0 +1,39 @@ +"""Pry โ€” Circuit Breaker router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Circuit Breaker"]) + +@router.post("/v1/breaker/status", tags=["Circuit Breaker"], summary="Get circuit breaker status") +async def breaker_status() -> dict[str, Any]: + return { + "success": True, + "data": {k: {"fails": v, "backoff": min(2**v, 60)} for k, v in _failures.items()}, + } + + +@router.post( + "/v1/breaker/reset", tags=["Circuit Breaker"], summary="Reset circuit breaker for a domain" +) +async def breaker_reset(domain: str = Body("")) -> dict[str, Any]: + if domain: + _failures.pop(domain, None) + else: + _failures.clear() + return {"success": True, "data": {"cleared": domain or "all"}} + + +_failures: dict[str, int] = {} diff --git a/routers/commerce.py b/routers/commerce.py new file mode 100644 index 0000000..8e001e0 --- /dev/null +++ b/routers/commerce.py @@ -0,0 +1,87 @@ +"""Pry โ€” Commerce router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Commerce"]) + +@router.post("/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify") +async def sync_commerce( + platform: str = Body(...), + products: list[dict[str, Any]] = Body(...), + credentials: dict[str, Any] = Body(...), +) -> dict[str, Any]: + """Sync scraped products to your e-commerce platform. + + Platforms: + - woocommerce: credentials = {"wp_url": "...", "consumer_key": "...", "consumer_secret": "..."} + - shopify: credentials = {"shop_url": "...", "access_token": "..."} + + Products are imported as drafts for review before publishing. + """ + from commerce_sync import sync_to_shopify, sync_to_woocommerce + + if platform == "woocommerce": + result = await sync_to_woocommerce( + products=products, + wp_url=credentials.get("wp_url", ""), + consumer_key=credentials.get("consumer_key", ""), + consumer_secret=credentials.get("consumer_secret", ""), + category_id=credentials.get("category_id", 0), + status=credentials.get("status", "draft"), + ) + elif platform == "shopify": + result = await sync_to_shopify( + products=products, + shop_url=credentials.get("shop_url", ""), + access_token=credentials.get("access_token", ""), + ) + else: + raise InvalidRequestError( + f"Unsupported platform: {platform}. Supported: woocommerce, shopify" + ) + + return {"success": result["success"], "data": result} + + +@router.get("/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms") +async def list_commerce_platforms() -> dict[str, Any]: + """List supported e-commerce platforms and their credential requirements.""" + return { + "success": True, + "data": { + "platforms": [ + { + "id": "woocommerce", + "name": "WooCommerce", + "credential_fields": [ + "wp_url", + "consumer_key", + "consumer_secret", + "category_id", + ], + "product_fields": ["name", "price", "description", "image_url"], + }, + { + "id": "shopify", + "name": "Shopify", + "credential_fields": ["shop_url", "access_token"], + "product_fields": ["name", "price", "description", "image_url"], + }, + ] + }, + } diff --git a/routers/compliance.py b/routers/compliance.py new file mode 100644 index 0000000..4cb7755 --- /dev/null +++ b/routers/compliance.py @@ -0,0 +1,57 @@ +"""Pry โ€” Compliance router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Compliance"]) + +@router.post("/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL") +async def compliance_check(url: str = Body(...)) -> dict[str, Any]: + """Run a full legal compliance check on a target URL. + + Analyzes: + - robots.txt crawl permissions + - Terms of Service classification (restrictive/permissive/moderate) + - Jurisdiction detection (GDPR, CCPA, LGPD) + - Sensitive data detection (PII, financial, health, contact) + + Returns a green/yellow/red risk score with recommendations. + """ + from compliance import run_compliance_check + + result = await run_compliance_check(url) + return {"success": True, "data": result} + + +@router.get("/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation") +async def compliance_docs() -> dict[str, Any]: + """Get information about the compliance check endpoint.""" + return { + "success": True, + "data": { + "description": "Legal compliance engine for web scraping targets", + "risk_levels": { + "green": "Low risk โ€” standard scraping practices", + "yellow": "Moderate risk โ€” proceed with caution", + "red": "High risk โ€” legal review required", + }, + "factors_checked": [ + "robots.txt crawl permissions", + "Terms of Service classification", + "Jurisdiction detection (GDPR/CCPA/LGPD)", + "Sensitive data categories (PII, financial, health, contact)", + ], + }, + } diff --git a/routers/config.py b/routers/config.py new file mode 100644 index 0000000..85a4781 --- /dev/null +++ b/routers/config.py @@ -0,0 +1,49 @@ +"""Pry โ€” Config router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from mconfig import PryConfig + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Config"]) + +@router.get("/v1/config", tags=["Config"], summary="Get current Pry configuration") +async def get_config() -> dict[str, Any]: + """Get current Pry configuration.""" + return {"success": True, "data": config.to_dict()} + + +@router.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime") +async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Update Pry configuration at runtime.""" + result = config.update(updates) + return {"success": True, "data": result} + + +@router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests") +async def enable_tor() -> dict[str, Any]: + """Enable Tor routing for all requests.""" + result = config.update( + {"tor": {"enabled": True}, "proxy": {"enabled": True, "url": "socks5://tor:9050"}} + ) + resp_data = { + "success": True, + "data": result, + "note": "Run 'docker compose --profile tor up -d' to start Tor container", + } + return resp_data + + +config = PryConfig() diff --git a/routers/costing.py b/routers/costing.py new file mode 100644 index 0000000..2f0a022 --- /dev/null +++ b/routers/costing.py @@ -0,0 +1,70 @@ +"""Pry โ€” Costing router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Costing"]) + +@router.get("/v1/costing/dashboard", tags=["Costing"], summary="Get cost analytics dashboard") +async def cost_dashboard() -> dict[str, Any]: + """Get the full cost analytics dashboard. + + Includes: + - Monthly spend breakdown by operation type + - Projected end-of-month cost + - Daily spend for last 7 days + - Cache efficiency metrics + - Smart schedule recommendations + """ + from costing import get_cost_dashboard + + return {"success": True, "data": get_cost_dashboard()} + + +@router.get("/v1/costing/usage", tags=["Costing"], summary="Get usage breakdown") +async def cost_usage( + year: int | None = None, + month: int | None = None, +) -> dict[str, Any]: + """Get detailed usage breakdown for a specific month.""" + from costing import get_monthly_usage + + return {"success": True, "data": get_monthly_usage(year, month)} + + +@router.post("/v1/costing/record", tags=["Costing"], summary="Record a usage event") +async def record_usage_endpoint( + operation: str = Body(...), + quantity: float = Body(1.0), + metadata: dict[str, Any] | None = Body(None), +) -> dict[str, Any]: + """Record a usage event for cost tracking. + + Operations: scrape_direct, scrape_flaresolverr, scrape_playwright, + crawl_page, llm_call, vision_call, extraction_css, bandwidth_mb + """ + from costing import record_usage + + result = record_usage(operation, metadata, quantity) + return {"success": True, "data": result} + + +@router.post("/v1/costing/costs", tags=["Costing"], summary="Update per-operation cost table") +async def update_costs(costs: dict[str, float] = Body(...)) -> dict[str, Any]: + """Update the per-operation cost table with custom prices.""" + from costing import update_cost_table + + result = await update_cost_table(costs) + return {"success": result["success"], "data": result} diff --git a/routers/crm.py b/routers/crm.py new file mode 100644 index 0000000..62ae33f --- /dev/null +++ b/routers/crm.py @@ -0,0 +1,113 @@ +"""Pry โ€” CRM router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["CRM"]) + +@router.post("/v1/crm/sync", tags=["CRM"], summary="Sync scraped data to CRM") +async def sync_crm( + platform: str = Body(...), + object_type: str = Body("lead"), + objects: list[dict[str, Any]] = Body(...), + credentials: dict[str, Any] = Body(...), +) -> dict[str, Any]: + """Sync scraped data to your CRM. + + Platforms: + - salesforce: credentials={"instance_url": "...", "access_token": "..."} + - hubspot: credentials={"api_key": "..."} + - pipedrive: credentials={"api_token": "...", "domain": "..."} + - close: credentials={"api_key": "..."} + + Object types vary by platform: + - Salesforce: Lead, Contact, Account, Opportunity + - HubSpot: contacts, companies, deals + - Pipedrive: person, organization, deal, lead + - Close: lead, contact + """ + from crm_sync import sync_to_close, sync_to_hubspot, sync_to_pipedrive, sync_to_salesforce + + platform_map = { + "salesforce": (sync_to_salesforce, ("instance_url", "access_token")), + "hubspot": (sync_to_hubspot, ("api_key",)), + "pipedrive": (sync_to_pipedrive, ("api_token",)), + "close": (sync_to_close, ("api_key",)), + } + + if platform not in platform_map: + raise InvalidRequestError( + f"Unsupported platform: {platform}. Supported: {list(platform_map.keys())}" + ) + + handler, required_fields = platform_map[platform] + for field in required_fields: + if not credentials.get(field): + raise InvalidRequestError(f"Missing required credential: {field}") + + if platform == "salesforce": + result = await handler( + objects, object_type, credentials["instance_url"], credentials["access_token"] + ) + elif platform == "hubspot": + result = await handler(objects, object_type, credentials["api_key"]) + elif platform == "pipedrive": + result = await handler( + objects, object_type, credentials["api_token"], credentials.get("domain", "") + ) + elif platform == "close": + result = await handler(objects, object_type, credentials["api_key"]) + else: + raise InvalidRequestError(f"Platform {platform} not handled") + + return {"success": result["success"], "data": result} + + +@router.get("/v1/crm/platforms", tags=["CRM"], summary="List supported CRM platforms") +async def list_crm_platforms() -> dict[str, Any]: + """List supported CRM platforms and their credential requirements.""" + return { + "success": True, + "data": { + "platforms": [ + { + "id": "salesforce", + "name": "Salesforce", + "objects": ["Lead", "Contact", "Account", "Opportunity"], + "credential_fields": ["instance_url", "access_token"], + }, + { + "id": "hubspot", + "name": "HubSpot", + "objects": ["contacts", "companies", "deals"], + "credential_fields": ["api_key"], + }, + { + "id": "pipedrive", + "name": "Pipedrive", + "objects": ["person", "organization", "deal", "lead"], + "credential_fields": ["api_token", "domain"], + }, + { + "id": "close", + "name": "Close.com", + "objects": ["lead", "contact"], + "credential_fields": ["api_key"], + }, + ] + }, + } diff --git a/routers/dashboard.py b/routers/dashboard.py new file mode 100644 index 0000000..6a6f09e --- /dev/null +++ b/routers/dashboard.py @@ -0,0 +1,64 @@ +"""Pry โ€” Dashboard router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from datetime import UTC, datetime + +from fastapi import APIRouter +from fastapi.responses import HTMLResponse + +from deps import automator, cache, ratelimiter + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Dashboard"]) + +@router.get("/dashboard", tags=["Dashboard"], summary="Pry health and performance dashboard") +async def dashboard() -> HTMLResponse: + cache_stats = cache.stats() + rate_stats = ratelimiter.get_stats() + html = f""" + +Pry โ€” Dashboard + +

๐Ÿ”ง Pry Dashboard

+

Scrape engine health and performance metrics

+
+

Cache Hit Rate

{cache_stats.get("hit_rate", 0)}%
{cache_stats.get("hits", 0)} hits / {cache_stats.get("size", 0)} entries
+

Rate Limit

{rate_stats.get("total_requests", 0)}
{rate_stats.get("active_ips", 0)} active IPs
+

Blocked

{rate_stats.get("total_blocked", 0)}
requests blocked
+

Sessions

{len(automator.sessions)}
active browser sessions
+
+ + + + + + + + +
EndpointMethodStatus
/v1/scrapePOSTโœ… Active
/v1/crawlPOSTโœ… Active
/v1/automatePOSTโœ… Active
/v1/batchPOSTโœ… Active
/v1/streamWebSocketโœ… Active
/v1/runPOSTโœ… Active
FlareSolverrProxyโœ… Connected
+

Pry v3.0.0 โ€” Generated {datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")}

+""" + return HTMLResponse(content=html) diff --git a/routers/email.py b/routers/email.py new file mode 100644 index 0000000..204c384 --- /dev/null +++ b/routers/email.py @@ -0,0 +1,76 @@ +"""Pry โ€” Email router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Email"]) + +@router.post("/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)") +async def scrape_email( + subject: str = Body(""), + body: str = Body(""), + sender: str = Body(""), +) -> dict[str, Any]: + """Extract structured data from an email subject and body. + + Auto-classifies as: order_confirmation, invoice, receipt, + shipping_notification, subscription, or other. + + Extracts: order numbers, amounts, tracking numbers, dates, addresses. + """ + from email_scraper import extract_email_data + + result = extract_email_data(subject, body, sender) + return {"success": True, "data": result} + + +@router.post("/v1/email/gmail", tags=["Email"], summary="Fetch and extract data from Gmail inbox") +async def fetch_gmail( + access_token: str = Body(...), + max_results: int = Body(20), + query: str = Body(""), + since_days: int = Body(7), +) -> dict[str, Any]: + """Connect to Gmail and extract structured data from emails. + + Requires a Gmail OAuth2 access token with scope: + https://www.googleapis.com/auth/gmail.readonly + + Extracts order confirmations, invoices, receipts, shipping notifications. + """ + from email_scraper import fetch_gmail_emails + + result = await fetch_gmail_emails(access_token, max_results, query, since_days) + return {"success": result.get("success", False), "data": result} + + +@router.post("/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox") +async def fetch_outlook( + access_token: str = Body(...), + max_results: int = Body(20), + query: str = Body(""), + since_days: int = Body(7), +) -> dict[str, Any]: + """Connect to Outlook/Office 365 and extract structured data from emails. + + Requires a Microsoft Graph OAuth2 access token with scope: + https://graph.microsoft.com/Mail.Read + + Extracts order confirmations, invoices, receipts, shipping notifications. + """ + from email_scraper import fetch_outlook_emails + + result = await fetch_outlook_emails(access_token, max_results, query, since_days) + return {"success": result.get("success", False), "data": result} diff --git a/routers/enrichment.py b/routers/enrichment.py new file mode 100644 index 0000000..bbc9ab9 --- /dev/null +++ b/routers/enrichment.py @@ -0,0 +1,65 @@ +"""Pry โ€” Enrichment router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Enrichment"]) + +@router.post( + "/v1/enrich", + tags=["Enrichment"], + summary="Enrich scraped data with company info, tech stack, social profiles", +) +async def enrich_data( + url: str = Body(...), + include_tech_stack: bool = Body(True), + include_social: bool = Body(True), + include_company: bool = Body(True), +) -> dict[str, Any]: + """Enrich a URL with supplemental business intelligence. + + Returns: + - Tech stack detection (CMS, framework, CDN, analytics, payments) + - Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.) + - Company information (email, phone, address, founded year, team size) + """ + from enrichment import enrich_url + + result = await enrich_url(url) + filtered: dict[str, Any] = {"url": url} + if include_tech_stack: + filtered["tech_stack"] = result.get("tech_stack") + if include_social: + filtered["social_profiles"] = result.get("social_profiles") + if include_company: + filtered["company_info"] = result.get("company_info") + + return {"success": True, "data": filtered} + + +@router.post( + "/v1/enrich/tech-stack", tags=["Enrichment"], summary="Detect technologies used on a website" +) +async def detect_tech(url: str = Body(...)) -> dict[str, Any]: + """Detect what technologies a website is built with. + + Detects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails), + frontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar), + payments (Stripe, PayPal). + """ + from enrichment import enrich_url + + result = await enrich_url(url) + return {"success": True, "data": result.get("tech_stack", {})} diff --git a/routers/execute.py b/routers/execute.py new file mode 100644 index 0000000..112e4a4 --- /dev/null +++ b/routers/execute.py @@ -0,0 +1,44 @@ +"""Pry โ€” Execute router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import InvalidRequestError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Execute"]) + +@router.post("/v1/run", tags=["Execute"], summary="Execute a Pryfile") +async def run_pryfile(path: str = Body("pry.yml")) -> dict[str, Any]: + from pryfile import Pryfile + + resolved = _safe_pryfile_path(path) + pf = Pryfile(resolved) + results = await pf.run_all(scraper) + return {"success": True, "data": {"jobs": results, "total": len(results)}} + + +def _safe_pryfile_path(path: str) -> str: + """Resolve and validate Pryfile path โ€” prevent directory traversal.""" + resolved = Path(path).resolve() + allowed = Path.cwd().resolve() + try: + resolved.relative_to(allowed) + except ValueError as e: + raise InvalidRequestError(f"Path must be inside {allowed}") from e + if not resolved.is_file(): + raise InvalidRequestError(f"File not found: {resolved}") + return str(resolved) diff --git a/routers/export.py b/routers/export.py new file mode 100644 index 0000000..5b42e48 --- /dev/null +++ b/routers/export.py @@ -0,0 +1,64 @@ +"""Pry โ€” Export router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import html +import logging +from datetime import UTC +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import ScrapeError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Export"]) + +@router.post("/v1/export", tags=["Export"], summary="Export scraped content in multiple formats") +async def export_data(url: str = Body(...), format: str = "json") -> dict[str, Any]: + """Export scraped content in multiple formats: json, csv, txt, rss. + Firecrawl only does markdown.""" + result = await scraper.scrape(url) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Export failed") + content = result.get("content", "") + title = result.get("title", url) + + if format == "json": + return { + "success": True, + "data": {"url": url, "title": title, "content": content, "format": "json"}, + } + elif format == "csv": + import csv + import io + + buf = io.StringIO() + w = csv.writer(buf) + w.writerow(["url", "title", "content"]) + w.writerow([url, title, content[:50000]]) + return {"success": True, "data": {"csv": buf.getvalue(), "format": "csv"}} + elif format == "rss": + from datetime import datetime + + safe_title = html.escape(title) + safe_url = html.escape(url) + rss = f""" + +Pry: {safe_title} +{safe_url} +Scraped content from {safe_url} +{safe_title}{safe_url} + +{datetime.now(UTC).strftime("%a, %d %b %Y %H:%M:%S UTC")} +""" + return {"success": True, "data": {"rss": rss, "format": "rss"}} + return {"success": True, "data": {"text": content[:100000], "format": "txt"}} diff --git a/routers/extraction.py b/routers/extraction.py new file mode 100644 index 0000000..dcde526 --- /dev/null +++ b/routers/extraction.py @@ -0,0 +1,256 @@ +"""Pry โ€” Extraction router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +import re +from typing import Any +from urllib.parse import urljoin, urlparse + +import httpx +from fastapi import APIRouter, Body + +from client import get_client +from deps import advanced, scraper +from errors import InvalidRequestError, ScrapeError +from extraction import JsonCssExtractionStrategy, extract_with_chunking +from extractor import SchemaExtractor + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Extraction"]) + +@router.post( + "/v1/extract-table", tags=["Extraction"], summary="Extract HTML tables as structured data" +) +async def extract_table(url: str = Body(...), table_index: int = 0) -> dict[str, Any]: + """Extract HTML tables from a page as structured data. + Firecrawl doesn't support table extraction at all.""" + import pandas as pd + + client = await get_client() + resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) + tables = pd.read_html(resp.text) + if table_index >= len(tables): + raise InvalidRequestError(f"Only {len(tables)} tables found") + df = tables[table_index] + return { + "success": True, + "data": { + "table_index": table_index, + "total_tables": len(tables), + "columns": list(df.columns), + "rows": df.to_dict(orient="records"), + "html": df.to_html(index=False), + }, + } + + +@router.post("/v1/links", tags=["Extraction"], summary="Analyze links on a page") +async def analyze_links(url: str = Body(...)) -> dict[str, Any]: + """Analyze all links on a page โ€” internal, external, broken, social. + Firecrawl only has basic map functionality.""" + html = "" + try: + client = await get_client() + resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) + html = resp.text + except httpx.HTTPError: + logger.warning("links_fetch_failed", extra={"url": url}) + base = urlparse(url).netloc + internal, external = set(), set() + for m in re.finditer(r'href=["\'](https?://[^"\']+)["\']', html): + link = m.group(1) + if urlparse(link).netloc == base: + internal.add(link.split("#")[0]) + else: + external.add(link.split("#")[0]) + for m in re.finditer(r'href=["\'](/[^"\']+)["\']', html): + internal.add(urljoin(url, m.group(1)).split("#")[0]) + social = advanced.find_social_links(html) + return { + "success": True, + "data": { + "url": url, + "internal_count": len(internal), + "external_count": len(external), + "internal": sorted(internal)[:50], + "external": sorted(external)[:50], + "social": social, + }, + } + + +@router.post("/v1/seo", tags=["Extraction"], summary="SEO analysis of a page") +async def analyze_seo(url: str = Body(...)) -> dict[str, Any]: + """SEO analysis of a page: title, description, headings, images, keywords, readability. + Firecrawl has zero SEO features.""" + client = await get_client() + resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) + html = resp.text + result = await scraper.scrape(url) + content = result.get("content", "") + + title = re.search(r"]*>(.*?)", html, re.I | re.S) + desc = re.search(r']*>(.*?)", html, re.I | re.S) + h2 = re.findall(r"]*>(.*?)", html, re.I | re.S) + imgs = re.findall(r"]*\salt=[\"\']([^\"\']*)[\"\'][^>]*>", html, re.I) + total_imgs = len(re.findall(r" dict[str, Any]: + """Extract Schema.org/JSON-LD structured data from a page.""" + client = await get_client() + resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) + html = resp.text + schemas = advanced.extract_schema(html) + return {"success": True, "data": {"url": url, "schemas": schemas}} + + +@router.post("/v1/emails", tags=["Extraction"], summary="Find email addresses on a page") +async def find_emails(url: str = Body(...)) -> dict[str, Any]: + """Find all email addresses on a page.""" + try: + client = await get_client() + resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) + html_text = resp.text + except (httpx.HTTPError, httpx.RequestError): + html_text = "" + + result = await scraper.scrape(url) + emails = advanced.find_emails(result.get("content", "")) + social = advanced.find_social_links(html_text) + return {"success": True, "data": {"url": url, "emails": emails, "social": social}} + + +@router.post("/v1/extract", tags=["Extraction"], summary="Extract structured fields from a URL") +async def extract_stable(data: dict[str, Any] = Body(...)) -> dict[str, Any]: + url = data.get("url", "") + fields = data.get("fields", {}) + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Extraction failed") + + ex = SchemaExtractor() + extracted = await ex.extract(result.get("content", ""), fields, mode="llm") + return {"success": True, "data": {"url": url, "fields": extracted}} + + +@router.post( + "/v1/extract/css", + tags=["Extraction"], + summary="Extract structured data with CSS selectors (no LLM)", +) +async def extract_css( + url: str = Body(...), + schema: dict[str, Any] = Body(...), + bypass_cloudflare: bool = Body(True), +) -> dict[str, Any]: + """Extract structured JSON from a URL using CSS selector schema. + + Schema format: + { + "name": "products", + "base_selector": ".product-card", + "fields": [ + {"name": "title", "selector": "h3", "type": "text"}, + {"name": "price", "selector": ".price", "type": "text", "transform": "float"}, + {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}, + {"name": "in_stock", "selector": ".stock", "type": "exists"}, + ] + } + """ + result = await scraper.scrape(url, {"bypass_cloudflare": bypass_cloudflare}) + html = result.get("raw_html", "") + + if not html: + client = await get_client() + resp = await client.get( + url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} + ) + html = resp.text + + if not html: + raise ScrapeError("No HTML content returned from scraper") + + strategy = JsonCssExtractionStrategy(schema) + data = strategy.extract(html) + + return { + "success": True, + "data": { + "schema": schema.get("name", "extracted"), + "count": len(data), + "items": data, + }, + } + + +@router.post("/v1/extract/llm", tags=["Extraction"], summary="Extract with LLM + chunking strategies") +async def extract_llm( + url: str = Body(...), + instruction: str = Body("Extract all key information from this content."), + schema: dict[str, Any] | None = Body(None), + chunk_strategy: str = Body("topic"), + query: str = Body(""), + top_k: int = Body(5), +) -> dict[str, Any]: + """Extract structured data using LLM with intelligent chunking. + + Chunks content by strategy (topic/sentence/regex), optionally filters + by relevance to query, then extracts from each chunk. + """ + + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + content = result.get("content", "") + if not content: + raise ScrapeError("No content returned from scraper") + + chunks = await extract_with_chunking( + content=content, + instruction=instruction, + schema=schema, + chunk_strategy=chunk_strategy, + query=query, + top_k=top_k, + ) + + return { + "success": True, + "data": { + "chunks": chunks, + "total_chunks": len(chunks), + "strategy": chunk_strategy, + }, + } + + +logger = logging.getLogger(__name__) diff --git a/routers/freshness.py b/routers/freshness.py new file mode 100644 index 0000000..510da7b --- /dev/null +++ b/routers/freshness.py @@ -0,0 +1,75 @@ +"""Pry โ€” Freshness router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Freshness"]) + +@router.post( + "/v1/freshness/check", + tags=["Freshness"], + summary="Check if content has changed since last scrape", +) +async def freshness_check( + url: str = Body(...), + content: str = Body(""), +) -> dict[str, Any]: + """Check if content has changed since the last scrape. + + Uses content fingerprinting (SHA256) to detect changes. + If no content is provided, does a quick HEAD check instead. + """ + from freshness import check_content_changed, quick_health_check, record_check_result + + if content: + result = await check_content_changed(url, content) + record_check_result(url, result["changed"]) + return {"success": True, "data": result} + + # Quick HEAD check + head = await quick_health_check(url) + return {"success": True, "data": head} + + +@router.post( + "/v1/freshness/frequency", + tags=["Freshness"], + summary="Get adaptive scrape frequency recommendation", +) +async def freshness_frequency( + url: str = Body(...), + base_interval: int = Body(60), +) -> dict[str, Any]: + """Get an adaptive scrape frequency recommendation based on content volatility. + + Volatile pages (frequent changes) get shorter intervals. + Stable pages get longer intervals to save costs. + """ + from freshness import calculate_adaptive_frequency + + result = calculate_adaptive_frequency(url, base_interval_minutes=base_interval) + return {"success": True, "data": result} + + +@router.get("/v1/freshness/dashboard", tags=["Freshness"], summary="Get content staleness dashboard") +async def freshness_dashboard() -> dict[str, Any]: + """Get the content staleness dashboard. + + Shows all tracked URLs, their last check time, age, and whether + they're stale (not checked in 24h). + """ + from freshness import get_staleness_dashboard + + return {"success": True, "data": get_staleness_dashboard()} diff --git a/routers/gdpr.py b/routers/gdpr.py new file mode 100644 index 0000000..ae2b79a --- /dev/null +++ b/routers/gdpr.py @@ -0,0 +1,123 @@ +"""Pry โ€” GDPR router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["GDPR"]) + +@router.post("/v1/gdpr/consent", tags=["GDPR"], summary="Record user consent for data processing") +async def record_consent_endpoint( + user_id: str = Body(...), + purpose: str = Body("data_collection"), + consent_given: bool = Body(True), + ip_address: str = Body(""), + user_agent: str = Body(""), +) -> dict[str, Any]: + """Record a user's consent for data processing (GDPR Art. 7). + + Stores: user hash, purpose, timestamp, IP, user agent. + Consent expires after 365 days. + """ + from gdpr import record_consent + + result = await record_consent(user_id, purpose, consent_given, ip_address, user_agent) + return {"success": result["success"], "data": result} + + +@router.get("/v1/gdpr/consent/{user_id}", tags=["GDPR"], summary="Check user consent status") +async def check_consent_endpoint( + user_id: str, + purpose: str = "data_collection", +) -> dict[str, Any]: + """Check if a user has given valid consent for data processing.""" + from gdpr import check_consent + + result = check_consent(user_id, purpose) + return {"success": True, "data": result} + + +@router.post("/v1/gdpr/consent/revoke", tags=["GDPR"], summary="Revoke user consent") +async def revoke_consent_endpoint( + user_id: str = Body(...), + purpose: str = Body("data_collection"), +) -> dict[str, Any]: + """Revoke a user's consent for a processing purpose (GDPR Art. 7(3)).""" + from gdpr import revoke_consent + + result = revoke_consent(user_id, purpose) + return {"success": True, "data": result} + + +@router.post( + "/v1/gdpr/deletion/request", tags=["GDPR"], summary="Request data deletion (right to erasure)" +) +async def request_deletion_endpoint( + user_id: str = Body(...), + reason: str = Body("user_request"), +) -> dict[str, Any]: + """Request deletion of all data associated with a user (GDPR Art. 17). + + Creates a deletion request that can be executed immediately or + after a holding period. + """ + from gdpr import request_deletion + + result = await request_deletion(user_id, reason) + return {"success": "error" not in result, "data": result} + + +@router.post("/v1/gdpr/deletion/execute", tags=["GDPR"], summary="Execute data deletion request") +async def execute_deletion_endpoint( + request_id: str = Body(...), +) -> dict[str, Any]: + """Execute a pending deletion request, removing all user data.""" + from gdpr import execute_deletion + + result = await execute_deletion(request_id) + if "error" in result: + raise NotFoundError(result["error"]) + return {"success": True, "data": result} + + +@router.get("/v1/gdpr/retention", tags=["GDPR"], summary="Get data retention policy") +async def get_retention_policy_endpoint() -> dict[str, Any]: + """Get the current data retention policy.""" + from gdpr import get_retention_policy + + return {"success": True, "data": get_retention_policy()} + + +@router.post( + "/v1/gdpr/retention/apply", + tags=["GDPR"], + summary="Apply retention policy to remove expired data", +) +async def apply_retention() -> dict[str, Any]: + """Apply the retention policy, removing all expired data.""" + from gdpr import apply_retention_policy + + result = await apply_retention_policy() + return {"success": True, "data": result} + + +@router.get("/v1/gdpr/audit", tags=["GDPR"], summary="Get compliance audit log") +async def get_audit_log_endpoint(days_back: int = 7) -> dict[str, Any]: + """Get the compliance audit log for the specified period.""" + from gdpr import get_audit_log + + events = get_audit_log(days_back) + return {"success": True, "data": {"events": events, "total": len(events)}} diff --git a/routers/integrations.py b/routers/integrations.py new file mode 100644 index 0000000..d5038b7 --- /dev/null +++ b/routers/integrations.py @@ -0,0 +1,137 @@ +"""Pry โ€” Integrations router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import InvalidRequestError, ScrapeError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Integrations"]) + +@router.get( + "/v1/destinations", + tags=["Integrations"], + summary="List supported data destinations", +) +async def list_destinations() -> dict[str, Any]: + """List all supported data destinations and their config requirements.""" + return { + "success": True, + "data": { + "destinations": [ + { + "id": "slack", + "name": "Slack", + "description": "Send data to a Slack channel via webhook", + "config_fields": ["webhook_url", "title"], + }, + { + "id": "googlesheets", + "name": "Google Sheets", + "description": "Write data to a Google Spreadsheet", + "config_fields": ["spreadsheet_id", "range", "credentials_json"], + }, + { + "id": "airtable", + "name": "Airtable", + "description": "Write records to an Airtable base", + "config_fields": ["base_id", "table_name", "api_key"], + }, + { + "id": "email", + "name": "Email", + "description": "Send data via email (SMTP or mailto link)", + "config_fields": ["recipient", "subject", "smtp_host", "smtp_user"], + }, + ] + }, + } + + +@router.post( + "/v1/destination/send", + tags=["Integrations"], + summary="Send scraped data to a business destination", +) +async def send_to_destination( + destination: str = Body(...), + data: dict[str, Any] = Body(...), + config: dict[str, Any] = Body(...), + url: str | None = Body(None), +) -> dict[str, Any]: + """Send extracted data to a business destination in one click. + + Destinations: + - slack: Send to Slack channel via webhook + - googlesheets: Write to Google Sheets (requires credentials) + - airtable: Write to Airtable base (requires API key) + - email: Send via email (requires SMTP config) + + Config varies by destination: + - slack: {"webhook_url": "https://hooks.slack.com/..."} + - googlesheets: {"spreadsheet_id": "...", "credentials_json": "..."} + - airtable: {"base_id": "...", "table_name": "Table 1", "api_key": "..."} + - email: {"recipient": "user@example.com", "subject": "Data Export"} + """ + from destinations import SUPPORTED_DESTINATIONS, dispatch + + if destination not in SUPPORTED_DESTINATIONS: + raise InvalidRequestError( + f"Unsupported destination: {destination}. Supported: {SUPPORTED_DESTINATIONS}" + ) + + result = await dispatch(destination, data, config) + return {"success": result["success"], "data": result} + + +@router.post( + "/v1/scrape-and-send", + tags=["Integrations"], + summary="Scrape a URL and send to a destination", +) +async def scrape_and_send( + url: str = Body(...), + destination: str = Body(...), + destination_config: dict[str, Any] = Body(...), +) -> dict[str, Any]: + """Scrape a URL and send the results to a business destination in one step. + + Combines /v1/scrape + /v1/destination/send into a single call. + Perfect for non-technical users who just want data in their tools. + """ + from destinations import SUPPORTED_DESTINATIONS, dispatch + + if destination not in SUPPORTED_DESTINATIONS: + raise InvalidRequestError( + f"Unsupported destination: {destination}. Supported: {SUPPORTED_DESTINATIONS}" + ) + + # Scrape + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + # Send to destination + send_result = await dispatch(destination, result.get("data", result), destination_config) + + return { + "success": send_result["success"], + "data": { + "url": url, + "destination": destination, + "scrape_status": result.get("status"), + "delivery": send_result, + }, + } diff --git a/routers/intelligence.py b/routers/intelligence.py new file mode 100644 index 0000000..a954f43 --- /dev/null +++ b/routers/intelligence.py @@ -0,0 +1,107 @@ +"""Pry โ€” Intelligence router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Intelligence"]) + +@router.post("/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot") +async def record_intel_snapshot( + competitor_id: str = Body(...), + competitor_name: str = Body(...), + url: str = Body(...), + fields: dict[str, Any] = Body(...), +) -> dict[str, Any]: + """Record a data snapshot for a competitor. + + Snapshots are stored with timestamps and used for trend analysis, + anomaly detection, and report generation. + """ + from intelligence import record_snapshot + + result = record_snapshot(competitor_id, competitor_name, url, fields) + return {"success": True, "data": result} + + +@router.get( + "/v1/intel/snapshots/{competitor_id}", tags=["Intelligence"], summary="Get competitor snapshots" +) +async def get_intel_snapshots( + competitor_id: str, + limit: int = 50, + since_hours: int | None = None, +) -> dict[str, Any]: + """Get historical snapshots for a competitor.""" + from intelligence import get_snapshots + + snapshots = get_snapshots(competitor_id, limit, since_hours) + return { + "success": True, + "data": {"competitor_id": competitor_id, "snapshots": snapshots, "count": len(snapshots)}, + } + + +@router.post( + "/v1/intel/analyze", tags=["Intelligence"], summary="Analyze competitor field for anomalies" +) +async def analyze_field( + competitor_id: str = Body(...), + field: str = Body("price"), +) -> dict[str, Any]: + """Analyze a specific field across a competitor's snapshots for anomalies. + + Returns statistics, z-score analysis, and anomaly detection. + """ + from intelligence import compute_field_statistics, detect_anomalies_numeric, get_snapshots + + snapshots = get_snapshots(competitor_id, limit=100) + stats = compute_field_statistics(snapshots, field) + + if ( + stats.get("has_history") + and stats.get("latest") is not None + and stats.get("previous") is not None + ): + numeric_vals = [ + s.get("fields", {}).get(field) + for s in snapshots + if isinstance(s.get("fields", {}).get(field), (int, float)) + ] + if len(numeric_vals) >= 3: + anomaly = detect_anomalies_numeric(numeric_vals[-1], numeric_vals[:-1]) + else: + anomaly = {"anomaly": False, "reason": "Insufficient numeric history"} + else: + anomaly = {"anomaly": False, "reason": "Insufficient data"} + + return {"success": True, "data": {"field": field, "statistics": stats, "anomaly": anomaly}} + + +@router.post( + "/v1/intel/report", tags=["Intelligence"], summary="Generate a competitive intelligence report" +) +async def generate_report( + competitors: list[dict[str, Any]] = Body(...), + days_back: int = Body(7), +) -> dict[str, Any]: + """Generate a competitive intelligence report for tracked competitors. + + Analyzes changes over the specified period and returns a structured report + with a natural-language summary. + """ + from intelligence import generate_weekly_report + + report = generate_weekly_report(competitors, days_back) + return {"success": True, "data": report} diff --git a/routers/jobs.py b/routers/jobs.py new file mode 100644 index 0000000..f84fa4b --- /dev/null +++ b/routers/jobs.py @@ -0,0 +1,52 @@ +"""Pry โ€” Jobs router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from fastapi import APIRouter + +from deps import queue +from errors import InvalidRequestError, NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Jobs"]) + +@router.get("/v1/job/{job_id}", tags=["Jobs"], summary="Get async job status and result") +async def get_job(job_id: str) -> dict[str, Any]: + """Get async job status and result.""" + job = await queue.get_job(job_id) + if not job: + raise NotFoundError("Job not found") + return {"success": True, "data": job} + + +@router.get("/v1/jobs", tags=["Jobs"], summary="List jobs in a Pryfile") +async def list_jobs(path: str = "pry.yml") -> dict[str, Any]: + from pryfile import Pryfile + + resolved = _safe_pryfile_path(path) + pf = Pryfile(resolved) + return {"success": True, "data": {"jobs": pf.list_jobs()}} + + +def _safe_pryfile_path(path: str) -> str: + """Resolve and validate Pryfile path โ€” prevent directory traversal.""" + resolved = Path(path).resolve() + allowed = Path.cwd().resolve() + try: + resolved.relative_to(allowed) + except ValueError as e: + raise InvalidRequestError(f"Path must be inside {allowed}") from e + if not resolved.is_file(): + raise InvalidRequestError(f"File not found: {resolved}") + return str(resolved) diff --git a/routers/marketplace.py b/routers/marketplace.py new file mode 100644 index 0000000..4dbe322 --- /dev/null +++ b/routers/marketplace.py @@ -0,0 +1,75 @@ +"""Pry โ€” Marketplace router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Marketplace"]) + +@router.post( + "/v1/actors/create", + tags=["Marketplace"], + summary="Create a new actor", +) +async def create_actor( + name: str = Body(...), + description: str = Body(...), + template_id: str = Body(""), + code: str = Body(""), + price_per_run: float = Body(0.0), + visibility: str = Body("private"), + schedule_cron: str = Body(""), + tags: list[str] | None = Body(None), +) -> dict[str, Any]: + """Create a new actor in the marketplace.""" + from actor_marketplace import ActorMarketplace, ActorVisibility + + m = ActorMarketplace() + actor = m.create( + name, + description, + template_id, + code, + price_per_run, + ActorVisibility(visibility), + schedule_cron, + tags or [], + ) + return {"success": True, "data": actor.to_dict()} + + +@router.get( + "/v1/actors", + tags=["Marketplace"], + summary="List actors in the marketplace", +) +async def list_actors(visibility: str = "", tag: str = "") -> dict[str, Any]: + """List actors. Filter by visibility (public/private) and tag.""" + from actor_marketplace import ActorMarketplace + + m = ActorMarketplace() + return {"success": True, "data": m.list(visibility, tag)} + + +@router.post( + "/v1/actors/{actor_id}/run", + tags=["Marketplace"], + summary="Run an actor", +) +async def run_actor(actor_id: str, inputs: dict[str, Any] | None = Body(None)) -> dict[str, Any]: + """Run an actor with the provided inputs.""" + from actor_marketplace import ActorMarketplace + + m = ActorMarketplace() + return await m.run(actor_id, inputs or {}) diff --git a/routers/monitoring.py b/routers/monitoring.py new file mode 100644 index 0000000..c249a3e --- /dev/null +++ b/routers/monitoring.py @@ -0,0 +1,116 @@ +"""Pry โ€” Monitoring router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Body + +from errors import NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Monitoring"]) + +@router.post("/v1/monitor", tags=["Monitoring"], summary="Create a scheduled content monitor") +async def create_monitor_endpoint( + name: str = Body(...), + url: str = Body(...), + schedule_cron: str = Body("0 */6 * * *"), + goal: str = Body(""), + webhook: str = Body(""), + use_llm: bool = Body(False), +) -> dict[str, Any]: + """Create a scheduled monitor that tracks content changes. + + Args: + name: Human-readable monitor name + url: Target URL to monitor + schedule_cron: Cron expression (default: every 6 hours) + goal: Natural language goal for meaningful-change detection + webhook: URL to notify on meaningful changes + use_llm: Use LLM for change judging (slower but smarter) + """ + from monitor import create_monitor + + monitor = await create_monitor(name, url, schedule_cron, goal, webhook, use_llm) + return {"success": True, "data": monitor} + + +@router.post("/v1/monitor/run", tags=["Monitoring"], summary="Run a single monitor check") +async def run_monitor_endpoint( + monitor_id: str = Body(...), +) -> dict[str, Any]: + """Execute a monitor check immediately (outside of schedule).""" + from monitor import run_monitor + + result = await run_monitor(monitor_id) + if "error" in result: + raise NotFoundError(result["error"]) + return {"success": True, "data": result} + + +@router.get("/v1/monitors", tags=["Monitoring"], summary="List all monitors") +async def list_monitors_endpoint() -> dict[str, Any]: + """List all registered monitors.""" + from monitor import list_monitors + + monitors = await list_monitors() + return {"success": True, "data": {"monitors": monitors, "total": len(monitors)}} + + +@router.delete("/v1/monitor/{monitor_id}", tags=["Monitoring"], summary="Delete a monitor") +async def delete_monitor_endpoint(monitor_id: str) -> dict[str, Any]: + """Delete a monitor and its snapshots.""" + from monitor import delete_monitor + + success = await delete_monitor(monitor_id) + if not success: + raise NotFoundError(f"Monitor not found: {monitor_id}") + return {"success": True, "data": {"monitor_id": monitor_id, "deleted": True}} + + +@router.post("/v1/monitor/check", tags=["Monitoring"], summary="Run all due monitors") +async def run_due_monitors() -> dict[str, Any]: + """Check all monitors and run any that are due based on their cron schedule.""" + import croniter + + from monitor import list_monitors, run_monitor + + monitors = await list_monitors() + now = datetime.now(UTC) + results = [] + for m in monitors: + last_run = m.get("last_run_at") + last_dt = datetime.fromisoformat(last_run) if last_run else datetime.min.replace(tzinfo=UTC) + + try: + cron = croniter.croniter(m["schedule_cron"], last_dt) + next_run = cron.get_next(datetime) + if next_run <= now: + result = await run_monitor(m["id"]) + results.append(result) + except Exception as e: # noqa: BLE001 + logger.warning( + "monitor_schedule_check_failed", extra={"monitor_id": m["id"], "error": str(e)} + ) + + return { + "success": True, + "data": { + "total_monitors": len(monitors), + "ran_count": len(results), + "results": results, + }, + } + + +logger = logging.getLogger(__name__) diff --git a/routers/parsing.py b/routers/parsing.py new file mode 100644 index 0000000..64aba8a --- /dev/null +++ b/routers/parsing.py @@ -0,0 +1,133 @@ +"""Pry โ€” Parsing router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +import httpx +from fastapi import APIRouter, Body +from pydantic import BaseModel + +from client import get_client +from deps import parser, scraper +from errors import ExternalServiceError, PryError, ScrapeError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Parsing"]) + +@router.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)") +async def parse_document(request: ParseRequest) -> dict[str, Any]: + """Parse a document (PDF, DOCX, image, CSV, JSON) to text.""" + try: + result = await parser.parse(request.url, request.timeout or 60) + return {"success": True, "data": result} + except PryError: + raise + except Exception as e: + raise ExternalServiceError(str(e)) from e + + +@router.post( + "/v1/markdown", tags=["Parsing"], summary="Generate markdown with content filtering strategies" +) +async def generate_markdown( + url: str = Body(...), + mode: str = Body("raw"), + query: str = Body(""), + threshold: float = Body(0.3), +) -> dict[str, Any]: + """Generate markdown with configurable content filtering. + + Modes: + - raw: Unfiltered markdown + - fit: Prune boilerplate (nav, ads, footers) + - bm25: Filter by BM25 relevance to query (requires query param) + """ + from markdown_gen import BM25ContentFilter, DefaultMarkdownGenerator, PruningContentFilter + + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + content = result.get("content", "") + if not content: + raise ScrapeError("No content scraped") + + if mode == "fit": + filter_strategy: PruningContentFilter | BM25ContentFilter | None = PruningContentFilter( + threshold=threshold + ) + elif mode == "bm25": + if not query: + filter_strategy = PruningContentFilter(threshold=threshold) + else: + filter_strategy = BM25ContentFilter(threshold=threshold) + else: + filter_strategy = None + + gen = DefaultMarkdownGenerator(content_filter=filter_strategy) + md_result = gen.generate(content, url=url, query=query) + + return { + "success": True, + "data": md_result, + } + + +@router.post("/v1/shadow-dom", tags=["Parsing"], summary="Extract content from Shadow DOM") +async def extract_shadow_dom( + url: str = Body(...), + flatten: bool = Body(True), +) -> dict[str, Any]: + """Scrape a page and extract content from Shadow DOM components. + + Useful for modern web apps built with Lit, web components, or + frameworks that use Shadow DOM encapsulation. + """ + from shadow_dom import ShadowDOMProcessor, has_shadow_dom + + result = await scraper.scrape(url, {"bypass_cloudflare": True, "js_render": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + html = result.get("raw_html", "") + if not html: + # Try to get via direct fetch + client = await get_client() + try: + resp = await client.get(url, timeout=30, follow_redirects=True) + html = resp.text + except (httpx.HTTPError, httpx.RequestError) as e: + raise ScrapeError("Could not fetch raw HTML") from e + + shadow_present = has_shadow_dom(html) + flat_html = "" + + if shadow_present and flatten: + processor = ShadowDOMProcessor() + flat_html = processor.process(html) + + return { + "success": True, + "data": { + "url": url, + "shadow_dom_detected": shadow_present, + "flattened": bool(flat_html), + "content": flat_html[:10000] if flat_html else html[:10000], + "raw_html_length": len(html), + "flat_html_length": len(flat_html) if flat_html else len(html), + }, + } + + +class ParseRequest(BaseModel): + url: str + timeout: int | None = 60 diff --git a/routers/pipeline.py b/routers/pipeline.py new file mode 100644 index 0000000..9571eb3 --- /dev/null +++ b/routers/pipeline.py @@ -0,0 +1,79 @@ +"""Pry โ€” Pipeline router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError +from pipeline import HOOK_POINTS, get_pipeline, run_pipeline + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Pipeline"]) + +@router.post("/v1/pipeline/hook", tags=["Pipeline"], summary="Register a hook function") +async def register_hook( + hook_point: str = Body(...), + javascript_fn: str = Body(...), +) -> dict[str, Any]: + """Register a JavaScript hook function at a pipeline hook point. + + Hook points: before_scrape, after_response, before_parse, after_parse, + before_extract, after_extract, before_return, on_error + """ + if hook_point not in HOOK_POINTS: + raise InvalidRequestError(f"Unknown hook point: {hook_point}. Valid: {HOOK_POINTS}") + + # For now, just acknowledge the registration + # In production, this would execute JS in a sandbox + return { + "success": True, + "data": { + "hook_point": hook_point, + "registered": True, + "message": f"Hook registered at {hook_point}. Note: custom JS hooks require a sandboxed runtime.", + }, + } + + +@router.get("/v1/pipeline/hooks", tags=["Pipeline"], summary="List all registered hooks") +async def list_pipeline_hooks() -> dict[str, Any]: + """List all registered hooks in the pipeline.""" + pipeline = get_pipeline() + return { + "success": True, + "data": { + "hooks": pipeline.list_hooks(), + "hook_points": HOOK_POINTS, + }, + } + + +@router.post("/v1/pipeline/run", tags=["Pipeline"], summary="Run pipeline hooks for testing") +async def run_pipeline_test( + hook_point: str = Body(...), + url: str = Body(""), + html: str = Body(""), +) -> dict[str, Any]: + """Run hooks at a given pipeline point for testing.""" + if hook_point not in HOOK_POINTS: + raise InvalidRequestError(f"Unknown hook point: {hook_point}") + + result = await run_pipeline(hook_point, url=url, html=html) + return { + "success": True, + "data": { + "hook_point": hook_point, + "result": {k: v for k, v in result.items() if k != "html"}, + "error_count": len(result.get("errors", [])), + }, + } diff --git a/routers/pipelines.py b/routers/pipelines.py new file mode 100644 index 0000000..022bcca --- /dev/null +++ b/routers/pipelines.py @@ -0,0 +1,112 @@ +"""Pry โ€” Pipelines router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError, NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Pipelines"]) + +@router.get( + "/v1/pipelines/steps", tags=["Pipelines"], summary="List all available pipeline step types" +) +async def list_step_types() -> dict[str, Any]: + """List all available step types for the visual pipeline builder. + + Each step type includes: name, icon, description, required inputs with types, + and expected outputs. A UI renders these as drag-and-drop blocks. + """ + from pipelines import STEP_TYPES + + return {"success": True, "data": {"step_types": STEP_TYPES, "total": len(STEP_TYPES)}} + + +@router.post("/v1/pipelines/validate", tags=["Pipelines"], summary="Validate a pipeline definition") +async def validate_pipeline_endpoint(pipeline: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Validate a pipeline definition for correctness.""" + from pipelines import validate_pipeline + + errors = validate_pipeline(pipeline) + return {"success": len(errors) == 0, "data": {"valid": len(errors) == 0, "errors": errors}} + + +@router.post("/v1/pipelines/run", tags=["Pipelines"], summary="Execute a pipeline") +async def run_pipeline_endpoint( + pipeline: dict[str, Any] = Body(...), + context: dict[str, Any] | None = Body(None), +) -> dict[str, Any]: + """Execute a pipeline definition. + + Steps run sequentially. Each step's output is available as + {{step_id.output_key}} in subsequent step templates. + + Example pipeline: + { + "name": "Monitor Competitor Pricing", + "steps": [ + {"id": "scrape_amazon", "type": "scrape", "inputs": {"url": "https://amazon.com/product"}}, + {"id": "extract", "type": "extract_css", "inputs": {"url": "{{scrape_amazon.url}}", "schema": {...}}}, + {"id": "quality", "type": "quality_check", "inputs": {"url": "{{scrape_amazon.url}}", "data": "{{extract.items}}"}}, + {"id": "notify", "type": "send_slack", "inputs": {"webhook_url": "https://hooks.slack.com/...", "message": "{{quality.quality_score}}"}}, + ] + } + """ + from pipelines import run_pipeline, validate_pipeline + + errors = validate_pipeline(pipeline) + if errors: + raise InvalidRequestError(f"Pipeline validation failed: {'; '.join(errors)}") + result = await run_pipeline(pipeline, context) + return {"success": not result["failed"], "data": result} + + +@router.post("/v1/pipelines/save", tags=["Pipelines"], summary="Save a pipeline definition") +async def save_pipeline_endpoint(pipeline: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Save a pipeline definition for later use.""" + from pipelines import save_pipeline + + result = save_pipeline(pipeline) + return {"success": result["success"], "data": result} + + +@router.get("/v1/pipelines", tags=["Pipelines"], summary="List saved pipelines") +async def list_pipelines_endpoint() -> dict[str, Any]: + """List all saved pipeline definitions.""" + from pipelines import list_pipelines + + pipelines = list_pipelines() + return {"success": True, "data": {"pipelines": pipelines, "total": len(pipelines)}} + + +@router.get("/v1/pipelines/{pipeline_id}", tags=["Pipelines"], summary="Get a saved pipeline") +async def get_pipeline_endpoint(pipeline_id: str) -> dict[str, Any]: + """Get a saved pipeline definition by ID.""" + from pipelines import get_pipeline + + result = get_pipeline(pipeline_id) + if not result: + raise NotFoundError(f"Pipeline not found: {pipeline_id}") + return {"success": True, "data": result} + + +@router.delete("/v1/pipelines/{pipeline_id}", tags=["Pipelines"], summary="Delete a saved pipeline") +async def delete_pipeline_endpoint(pipeline_id: str) -> dict[str, Any]: + """Delete a saved pipeline definition.""" + from pipelines import delete_pipeline + + success = delete_pipeline(pipeline_id) + if not success: + raise NotFoundError(f"Pipeline not found: {pipeline_id}") + return {"success": True, "data": {"deleted": True}} diff --git a/routers/proxy.py b/routers/proxy.py new file mode 100644 index 0000000..92fce5b --- /dev/null +++ b/routers/proxy.py @@ -0,0 +1,154 @@ +"""Pry โ€” Proxy router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Proxy"]) + +@router.get("/v1/proxy/providers", tags=["Proxy"], summary="List all available proxy providers") +async def list_proxy_providers() -> dict[str, Any]: + """List free + premium proxy providers with affiliate details.""" + from proxy_manager import ProxyManager + + return {"success": True, "data": ProxyManager().list_providers()} + + +@router.post("/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider") +async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]: + """Get the affiliate signup URL for a proxy provider. + + Records a click for revenue tracking. The user can then sign up at that URL + and come back to configure credentials. + """ + from proxy_manager import ProxyManager + + pm = ProxyManager() + url = pm.get_signup_link(provider) + return {"success": True, "data": {"signup_url": url, "provider": provider}} + + +@router.get("/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals") +async def list_proxy_referrals() -> dict[str, Any]: + """Return the curated proxy provider affiliate catalog (proxy_referrals.py). + + This is the marketing-friendly subset: commission, promo codes, tier + (premium/standard/budget), and referral URLs. Useful for a "Sign up via Pry + and we get a cut" UI, or for showing users premium options when their free + proxy fails. + + The full connection metadata (host, port, auth) lives in /v1/proxy/providers. + """ + from proxy_manager import ProxyManager + + pm = ProxyManager() + return { + "success": True, + "data": { + "providers": pm.list_proxy_referrals(), + "summary": pm.get_proxy_referral_summary(), + }, + } + + +@router.get( + "/v1/proxy/referrals/{tag}", tags=["Proxy"], summary="Get a single proxy provider referral" +) +async def get_proxy_referral(tag: str) -> dict[str, Any]: + """Return the affiliate info for a single proxy provider by tag.""" + from proxy_manager import ProxyManager + + pm = ProxyManager() + info = pm.get_proxy_referral(tag) + if not info: + return {"success": False, "error": f"No proxy referral found for tag: {tag}"} + return {"success": True, "data": {"tag": tag, **info}} + + +@router.post("/v1/proxy/configure", tags=["Proxy"], summary="Configure proxy credentials") +async def proxy_configure( + provider: str = Body(...), + username: str = Body(""), + password: str = Body(""), + api_key: str = Body(""), + proxy_url: str = Body(""), +) -> dict[str, Any]: + """Configure credentials for a premium proxy provider. + + After signing up via /v1/proxy/signup, the user provides their credentials here. + """ + from proxy_manager import ProxyManager + + pm = ProxyManager() + creds = {"username": username, "password": password, "api_key": api_key, "proxy_url": proxy_url} + creds = {k: v for k, v in creds.items() if v} + result = pm.select_provider(provider, creds) + return {"success": result["success"], "data": result} + + +@router.get("/v1/proxy/test", tags=["Proxy"], summary="Test the active proxy") +async def proxy_test() -> dict[str, Any]: + """Test the currently configured proxy and return its public IP.""" + from proxy_manager import ProxyManager + + pm = ProxyManager() + proxy_url = pm.get_proxy_url() + if not proxy_url: + return {"success": True, "data": {"active": False, "message": "No proxy configured"}} + result = pm.test_proxy(proxy_url) + return {"success": True, "data": {"active": True, **result}} + + +@router.get("/v1/proxy/status", tags=["Proxy"], summary="Get current proxy status") +async def proxy_status() -> dict[str, Any]: + """Get current proxy configuration and available providers.""" + from proxy_manager import ProxyManager + + pm = ProxyManager() + return { + "success": True, + "data": { + "active_config": pm.active_config.__dict__, + "configured_providers": list(pm.credentials.keys()), + "available_providers": pm.list_providers(), + }, + } + + +@router.post("/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block") +async def proxy_recommend(last_error: str = Body("")) -> dict[str, Any]: + """After a scrape fails with anti-bot detection, get a recommendation + for which premium proxy provider to sign up with.""" + from proxy_manager import ProxyManager + + pm = ProxyManager() + rec = pm.get_recommendation(last_error) + return {"success": True, "data": rec} + + +@router.get("/v1/proxy/clicks", tags=["Proxy"], summary="Get recent proxy referral clicks") +async def proxy_clicks(days_back: int = 30) -> dict[str, Any]: + """Get recent proxy referral clicks for revenue tracking.""" + from proxy_manager import ProxyManager + + pm = ProxyManager() + clicks = pm.get_recent_clicks(days_back=days_back) + return { + "success": True, + "data": { + "total_clicks": len(clicks), + "days_back": days_back, + "clicks": clicks, + }, + } diff --git a/routers/quality.py b/routers/quality.py new file mode 100644 index 0000000..bd2f472 --- /dev/null +++ b/routers/quality.py @@ -0,0 +1,89 @@ +"""Pry โ€” Quality router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import json +import logging +import os +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Quality"]) + +@router.post( + "/v1/quality/check", tags=["Quality"], summary="Run data quality check on extraction results" +) +async def quality_check( + url: str = Body(...), + data: dict[str, Any] = Body(...), + schema: dict[str, Any] | None = Body(None), + expected_types: dict[str, str] | None = Body(None), +) -> dict[str, Any]: + """Run a full data quality check on extraction results. + + Metrics: + - Completeness: what % of expected fields have values + - Schema adherence: field types match expectations + - Freshness: how old is the data + - Anomaly detection: what changed since last extraction + + Use this to validate data BEFORE sending to downstream systems. + """ + from quality import run_quality_check + + # Convert string type names to actual types + type_map: dict[str, type] = { + "str": str, + "int": int, + "float": float, + "bool": bool, + "list": list, + "dict": dict, + "None": type(None), + } + resolved_types: dict[str, type] | None = None + if expected_types: + resolved_types = {} + for field, type_name in expected_types.items(): + resolved_types[field] = type_map.get(type_name, str) + + result = await run_quality_check( + url=url, + data=data, + schema=schema, + expected_types=resolved_types, + ) + return {"success": True, "data": result} + + +@router.get( + "/v1/quality/stats", tags=["Quality"], summary="Get quality statistics for all checked URLs" +) +async def quality_stats() -> dict[str, Any]: + """Get aggregate quality statistics across all checked URLs.""" + from quality import QUALITY_DIR + + stats: list[dict[str, Any]] = [] + for path in sorted(QUALITY_DIR.glob("*.json"), key=os.path.getmtime, reverse=True)[:50]: + try: + data = json.loads(path.read_text()) + stats.append( + { + "url": data.get("url", ""), + "checked_at": data.get("checked_at", ""), + "size_bytes": path.stat().st_size, + } + ) + except (json.JSONDecodeError, OSError): + continue + + return {"success": True, "data": {"total_checked": len(stats), "recent": stats}} diff --git a/routers/reconciliation.py b/routers/reconciliation.py new file mode 100644 index 0000000..20cacad --- /dev/null +++ b/routers/reconciliation.py @@ -0,0 +1,71 @@ +"""Pry โ€” Reconciliation router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Reconciliation"]) + +@router.post( + "/v1/reconcile", + tags=["Reconciliation"], + summary="Reconcile records from multiple sources into unified entities", +) +async def reconcile_endpoint( + records: list[dict[str, Any]] = Body(...), + vertical: str = Body("product"), + threshold: float = Body(0.7), +) -> dict[str, Any]: + """Reconcile records from multiple sources into matched entities. + + Matches records across sources using identity field similarity, + normalizes to a unified vertical schema, and returns entity groups + with confidence scores. + + Verticals: product, job, real_estate, review + """ + from reconciliation import VERTICAL_SCHEMAS, reconcile + + if vertical not in VERTICAL_SCHEMAS: + raise InvalidRequestError( + f"Unknown vertical: {vertical}. Supported: {list(VERTICAL_SCHEMAS.keys())}" + ) + + result = await reconcile(records, vertical, threshold) + return {"success": True, "data": result} + + +@router.get( + "/v1/reconcile/schemas", + tags=["Reconciliation"], + summary="List supported reconciliation schemas", +) +async def list_schemas() -> dict[str, Any]: + """List all supported vertical schemas for entity reconciliation.""" + from reconciliation import VERTICAL_SCHEMAS + + schemas = {} + for key, schema in VERTICAL_SCHEMAS.items(): + schemas[key] = { + "name": schema["name"], + "fields": { + k: {fk: fv for fk, fv in v.items() if fk != "type"} + for k, v in schema["fields"].items() + }, + "identity_fields": schema["identity_fields"], + } + + return {"success": True, "data": {"schemas": schemas, "total": len(schemas)}} diff --git a/routers/recorder.py b/routers/recorder.py new file mode 100644 index 0000000..67653fd --- /dev/null +++ b/routers/recorder.py @@ -0,0 +1,45 @@ +"""Pry โ€” Recorder router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from pryextras import recorder + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Recorder"]) + +@router.post("/v1/record/start", tags=["Recorder"], summary="Start recording browser actions") +async def start_recording(session_id: str = Body(...)) -> dict[str, Any]: + recorder.start(session_id) + return {"success": True, "data": {"session_id": session_id, "status": "recording"}} + + +@router.post("/v1/record/step", tags=["Recorder"], summary="Record a browser action step") +async def record_step( + session_id: str = Body(...), action: str = Body(...), selector: str = "", value: str = "" +) -> dict[str, Any]: + recorder.record(session_id, action, selector, value) + return {"success": True, "data": {"recorded": len(recorder._recordings.get(session_id, []))}} + + +@router.post("/v1/record/export", tags=["Recorder"], summary="Export recording as script") +async def export_recording(session_id: str = Body(...), fmt: str = "json") -> dict[str, Any]: + script = recorder.export(session_id, fmt) + return {"success": True, "data": {"script": script, "format": fmt}} + + +@router.post("/v1/record/clear", tags=["Recorder"], summary="Clear recorded actions") +async def clear_recording(session_id: str = Body(...)) -> dict[str, Any]: + recorder.clear(session_id) + return {"success": True, "data": {"status": "cleared"}} diff --git a/routers/referrals.py b/routers/referrals.py new file mode 100644 index 0000000..9a8b1e5 --- /dev/null +++ b/routers/referrals.py @@ -0,0 +1,79 @@ +"""Pry โ€” Referrals router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Referrals"]) + +@router.get( + "/v1/referrals/catalog", + tags=["Referrals"], + summary="Get all available referral/affiliate programs", +) +async def get_referral_catalog(category: str = "") -> dict[str, Any]: + """List all referral programs Pry supports. + + 60+ providers across categories: LLM, hosting, domains, CDN, email, + monitoring, proxies, voice, media, devtools, search, CAPTCHA. + """ + from referrals import ReferralTracker + + return {"success": True, "data": ReferralTracker().get_catalog(category)} + + +@router.get( + "/v1/referrals/stats", tags=["Referrals"], summary="Get referral click and conversion stats" +) +async def get_referral_stats(days_back: int = 30) -> dict[str, Any]: + """Get referral tracking statistics for the last N days.""" + from referrals import ReferralTracker + + return {"success": True, "data": ReferralTracker().get_stats(days_back)} + + +@router.post("/v1/referrals/click", tags=["Referrals"], summary="Record a referral link click") +async def record_referral_click( + provider_tag: str = Body(...), + source: str = Body("api"), + user_id: str = Body(""), +) -> dict[str, Any]: + """Record when a user clicks a referral link. Returns tracking ID.""" + from referrals import PROVIDER_CATALOG, ReferralTracker + + url = "" + for _cat, providers in PROVIDER_CATALOG.items(): + for p in providers: + if p.get("tag") == provider_tag: + url = p["url"] + break + if url: + break + if not url: + return {"success": False, "error": f"Unknown provider: {provider_tag}"} + click_id = ReferralTracker().record_click(provider_tag, url, source, user_id) + return {"success": True, "data": {"click_id": click_id, "url": url}} + + +@router.post("/v1/referrals/convert", tags=["Referrals"], summary="Record a referral conversion") +async def record_referral_conversion( + click_id: str = Body(...), + revenue_usd: float = Body(0.0), + notes: str = Body(""), +) -> dict[str, Any]: + """Record that a referral click resulted in a conversion.""" + from referrals import ReferralTracker + + success = ReferralTracker().record_conversion(click_id, revenue_usd, notes) + return {"success": success} diff --git a/routers/reports.py b/routers/reports.py new file mode 100644 index 0000000..0b0345c --- /dev/null +++ b/routers/reports.py @@ -0,0 +1,68 @@ +"""Pry โ€” Reports router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body +from fastapi.responses import HTMLResponse + +from errors import InvalidRequestError, NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Reports"]) + +@router.post( + "/v1/reports/generate", + tags=["Reports"], + summary="Generate a white-label report from scraped data", +) +async def generate_report_endpoint( + report_type: str = Body(...), + data: dict[str, Any] = Body(...), + branding: dict[str, Any] | None = Body(None), + output_format: str = Body("html"), +) -> dict[str, Any]: + """Generate a white-label report from scraped data. + + Report types: + - competitive_analysis: Competitor pricing and activity overview + - price_monitor: Product price change tracking with visual indicators + - seo_audit: SEO element analysis with change detection + - content_tracker: Content change monitoring across pages + + Branding (optional): {"agency_name": "...", "brand_color": "#hex", "logo_url": "..."} + """ + from reports import generate_report + + result = generate_report(report_type, data, branding, output_format) + if "error" in result: + raise InvalidRequestError(result["error"]) + return {"success": True, "data": result} + + +@router.get("/v1/reports", tags=["Reports"], summary="List generated reports") +async def list_reports_endpoint() -> dict[str, Any]: + """List all previously generated reports.""" + from reports import list_reports + + reports = list_reports() + return {"success": True, "data": {"reports": reports, "total": len(reports)}} + + +@router.get("/v1/report/{report_id}", tags=["Reports"], summary="Get a generated report") +async def get_report(report_id: str) -> Any: + """Get the HTML content of a generated report.""" + from reports import REPORTS_DIR + + for path in REPORTS_DIR.glob(f"{report_id}_*.html"): + return HTMLResponse(content=path.read_text()) + raise NotFoundError(f"Report not found: {report_id}") diff --git a/routers/review.py b/routers/review.py new file mode 100644 index 0000000..aa58d1d --- /dev/null +++ b/routers/review.py @@ -0,0 +1,151 @@ +"""Pry โ€” Review router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import NotFoundError, ScrapeError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Review"]) + +@router.post("/v1/review/submit", tags=["Review"], summary="Submit extracted data for human review") +async def review_submit( + data: dict[str, Any] = Body(...), + url: str = Body(...), + schema_name: str | None = Body(None), + confidence_score: float = Body(0.0), + flagged_fields: list[dict[str, Any]] | None = Body(None), +) -> dict[str, Any]: + """Submit extracted data for human review before delivery. + + Use this when extraction confidence is low or anomalies were detected. + Data will be held in the review queue until approved or rejected. + """ + from review import submit_for_review + + result = await submit_for_review( + data=data, + extraction_url=url, + schema_name=schema_name, + confidence_score=confidence_score, + flagged_fields=flagged_fields, + ) + return {"success": True, "data": result} + + +@router.post("/v1/review/{review_id}/approve", tags=["Review"], summary="Approve a review item") +async def review_approve( + review_id: str, + reviewer: str = Body("api"), + notes: str = Body(""), +) -> dict[str, Any]: + """Approve a review item, allowing data to proceed to delivery.""" + from review import approve_review + + result = await approve_review(review_id, reviewer, notes) + if "error" in result: + raise NotFoundError(result["error"]) + return {"success": True, "data": result} + + +@router.post("/v1/review/{review_id}/reject", tags=["Review"], summary="Reject a review item") +async def review_reject( + review_id: str, + reviewer: str = Body("api"), + notes: str = Body(""), +) -> dict[str, Any]: + """Reject a review item, blocking data delivery.""" + from review import reject_review + + result = await reject_review(review_id, reviewer, notes) + if "error" in result: + raise NotFoundError(result["error"]) + return {"success": True, "data": result} + + +@router.get("/v1/reviews", tags=["Review"], summary="List reviews in the queue") +async def list_reviews(status: str | None = None) -> dict[str, Any]: + """List reviews, optionally filtered by status (pending/approved/rejected).""" + from review import get_review_queue + + reviews = get_review_queue(status) + return {"success": True, "data": {"reviews": reviews, "total": len(reviews)}} + + +@router.get("/v1/review/{review_id}", tags=["Review"], summary="Get review details") +async def get_review(review_id: str) -> dict[str, Any]: + """Get full details of a review item including the data payload.""" + from review import get_review_detail + + result = get_review_detail(review_id) + if not result: + raise NotFoundError(f"Review not found: {review_id}") + return {"success": True, "data": result} + + +@router.post( + "/v1/extract-with-review", + tags=["Review"], + summary="Extract with automatic human review routing", +) +async def extract_with_review( + url: str = Body(...), + schema: dict[str, Any] | None = Body(None), + expected_types: dict[str, str] | None = Body(None), + slack_webhook: str = Body(""), + auto_approve_threshold: float = Body(0.8), + auto_reject_threshold: float = Body(0.2), +) -> dict[str, Any]: + """Extract data with automatic quality check and human review routing. + + High-confidence results are auto-approved. + Low-confidence results are auto-rejected. + Medium-confidence results go to the human review queue with Slack notification. + """ + from quality import run_quality_check + from review import auto_review_threshold + + # Scrape + scrape_result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if scrape_result.get("status") != "ok": + raise ScrapeError(scrape_result.get("error") or "Scrape failed") + + data = scrape_result + + # Quality check + quality = await run_quality_check( + url=url, + data=data, + schema=schema, + expected_types=None, + ) + + # Auto-route + decision = await auto_review_threshold( + data=data, + extraction_url=url, + quality_result=quality, + slack_webhook=slack_webhook, + auto_approve_threshold=auto_approve_threshold, + auto_reject_threshold=auto_reject_threshold, + ) + + return { + "success": True, + "data": { + "decision": decision, + "quality": {k: v for k, v in quality.items() if k != "url"}, + }, + } diff --git a/routers/scraping_api.py b/routers/scraping_api.py new file mode 100644 index 0000000..87bfacc --- /dev/null +++ b/routers/scraping_api.py @@ -0,0 +1,179 @@ +"""Pry โ€” Scraping router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any +from urllib.parse import urljoin + +import httpx +import pydantic +from fastapi import APIRouter, Body + +from client import get_client +from deps import scraper +from errors import ScrapeError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Scraping"]) + +@router.post( + "/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system" +) +async def ultimate_scrape( + url: str = Body(..., embed=True), +) -> dict[str, Any]: + """Scrape any URL using Pry's ultimate 10-tier anti-detection system. + + Automatically tries: direct โ†’ cloudscraper โ†’ FlareSolverr โ†’ + undetected-chromedriver โ†’ Playwright โ†’ Googlebot โ†’ Archive.org โ†’ Google Cache + + Returns the first successful result with the method used. + """ + from ultimate_scraper import UltimateScraper + + s = UltimateScraper() + result = await s.scrape(url) + if result.get("status") != "ok": + raise ScrapeError(result.get("error", "All bypass methods failed")) + + return { + "success": True, + "data": { + "url": url, + "method": result.get("method", "unknown"), + "content": result.get("content", "")[:50000], + "content_length": len(result.get("content", "")), + }, + } + + +@router.post( + "/v1/capture/network", + tags=["Scraping"], + summary="Extract API calls and network patterns from a page", +) +async def capture_network( + url: str = Body(...), +) -> dict[str, Any]: + """Extract API calls, GraphQL queries, and network patterns from a page. + + Useful for understanding how SPAs load data and finding hidden API endpoints. + """ + from network import ( + extract_api_calls_from_html, + extract_graphql_queries, + extract_json_ld, + extract_nextjs_props, + extract_nuxt_state, + ) + + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + html = result.get("raw_html", "") + if not html: + client = await get_client() + try: + resp = await client.get( + url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} + ) + html = resp.text + except (httpx.HTTPError, httpx.RequestError): + raise ScrapeError("Could not fetch raw HTML") from None + + return { + "success": True, + "data": { + "url": url, + "api_calls": extract_api_calls_from_html(html), + "graphql_queries": extract_graphql_queries(html), + "json_ld": extract_json_ld(html), + "nextjs_props": extract_nextjs_props(html) is not None, + "nuxt_state": extract_nuxt_state(html) is not None, + }, + } + + +@router.post( + "/v1/crawl/adaptive", + tags=["Scraping"], + summary="Crawl with adaptive stopping based on content relevance", +) +async def adaptive_crawl( + url: str = Body(...), + query: str = Body(""), + max_pages: int = Body(50), + max_depth: int = Body(3), + relevance_threshold: float = Body(0.3), +) -> dict[str, Any]: + """Crawl a website with adaptive stopping. + + Uses information foraging theory to decide when to stop: + - Stops when content relevance drops below threshold + - Stops when information gain diminishes + - Respects max_pages and max_depth limits + - Ideal for targeted data collection (pricing, docs, products) + """ + from adaptive import AdaptiveCrawler + + crawler = AdaptiveCrawler( + max_pages=max_pages, + max_depth=max_depth, + relevance_threshold=relevance_threshold, + ) + + pages = [] + to_visit = [(url, 0)] + visited_urls: set[str] = set() + + while to_visit: + current_url, depth = to_visit.pop(0) + + if current_url in visited_urls: + continue + visited_urls.add(current_url) + + try: + result = await scraper.scrape(current_url, {"bypass_cloudflare": True}) + content = result.get("content", "") or "" + except pydantic.ValidationError as e: + logger.warning( + "adaptive_crawl_page_failed", extra={"url": current_url, "error": str(e)} + ) + continue + + decision = await crawler.should_continue(current_url, content, depth, query=query) + pages.append({"url": current_url, "depth": depth, "decision": decision}) + + if not decision["continue"]: + break + + if depth < max_depth: + links = await scraper.map_urls(current_url, {"limit": 10}) + for link in links: + full_url = urljoin(current_url, link) + if full_url not in visited_urls: + to_visit.append((full_url, depth + 1)) + + return { + "success": True, + "data": { + "url": url, + "query": query, + "pages": pages, + "total_pages": len(pages), + "stats": crawler.get_stats(), + }, + } + + +logger = logging.getLogger(__name__) diff --git a/routers/seo.py b/routers/seo.py new file mode 100644 index 0000000..f352581 --- /dev/null +++ b/routers/seo.py @@ -0,0 +1,64 @@ +"""Pry โ€” SEO router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["SEO"]) + +@router.post("/v1/seo/analyze", tags=["SEO"], summary="Analyze SEO elements from a URL") +async def seo_analyze(url: str = Body(...)) -> dict[str, Any]: + """Analyze all SEO elements from a URL. + + Returns: title, meta description, keywords, headings (H1/H2), + canonical, OG tags, Twitter cards, word count, link counts, + schema markup, hreflang tags. + """ + from seo_monitor import analyze_seo + + result = await analyze_seo(url) + return {"success": "error" not in result, "data": result} + + +@router.post("/v1/seo/track", tags=["SEO"], summary="Track SEO changes since last scan") +async def seo_track(url: str = Body(...)) -> dict[str, Any]: + """Track SEO changes since the last scan of this URL. + + Compares current SEO elements to previous snapshot and reports + what changed (title, description, headings, etc.). + """ + from seo_monitor import track_seo_changes + + result = await track_seo_changes(url) + return {"success": "error" not in result, "data": result} + + +@router.post("/v1/seo/keywords", tags=["SEO"], summary="Analyze keyword presence in URL content") +async def seo_keywords( + url: str = Body(...), + keywords: list[str] = Body(...), +) -> dict[str, Any]: + """Analyze which keywords a URL targets. + + Checks each keyword for: + - Presence in title tag + - Presence in H1 headings + - Presence in meta description + - Frequency in body content + - Keyword density percentage + """ + from seo_monitor import get_seo_keyword_insights + + result = await get_seo_keyword_insights(url, keywords) + return {"success": "error" not in result, "data": result} diff --git a/routers/sessions.py b/routers/sessions.py new file mode 100644 index 0000000..f1f727f --- /dev/null +++ b/routers/sessions.py @@ -0,0 +1,134 @@ +"""Pry โ€” Sessions router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Body + +from deps import automator +from errors import NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Sessions"]) + +@router.post("/v1/session/create", tags=["Sessions"], summary="Create a persistent browser session") +async def create_session(url: str = Body(...), persist: bool = True) -> dict[str, Any]: + """Create a persistent browser session.""" + session_id = await automator.create_session(url, persist=persist) + + if persist: + from sessions import save_session + + await save_session( + session_id=session_id, + cookies=[], + metadata={"url": url, "created_at": datetime.now(UTC).isoformat()}, + ) + + return {"success": True, "data": {"session_id": session_id, "persist": persist}} + + +@router.post( + "/v1/session/destroy", tags=["Sessions"], summary="Destroy a browser session with optional save" +) +async def destroy_session( + session_id: str = Body(...), + save_state: bool = Body(False), +) -> dict[str, Any]: + """Destroy a browser session. Optionally save state first.""" + from sessions import delete_session, save_session + + if save_state: + state = await automator.get_session_state(session_id) + if state: + await save_session( + session_id=session_id, + cookies=state.get("cookies", []), + local_storage=state.get("local_storage", {}), + metadata={"destroyed_at": datetime.now(UTC).isoformat()}, + ) + + success = await automator.destroy_session(session_id) + if not save_state: + await delete_session(session_id) + + return {"success": True, "data": {"session_id": session_id, "destroyed": success}} + + +@router.get("/v1/sessions", tags=["Sessions"], summary="List all persistent sessions") +async def list_sessions() -> dict[str, Any]: + """List all persistent sessions (active + saved).""" + from sessions import list_sessions as list_saved_sessions + + active = automator.list_sessions() + saved = await list_saved_sessions() + + return { + "success": True, + "data": { + "active": active, + "saved": saved, + "total_active": len(active), + "total_saved": len(saved), + }, + } + + +@router.post("/v1/session/save", tags=["Sessions"], summary="Save current session state to disk") +async def save_session_state( + session_id: str = Body(...), +) -> dict[str, Any]: + """Save a session's current state (cookies, storage) to disk.""" + from sessions import save_session as save_session_to_disk + + state = await automator.get_session_state(session_id) + if not state: + raise NotFoundError(f"Session not found: {session_id}") + + await save_session_to_disk( + session_id=session_id, + cookies=state.get("cookies", []), + local_storage=state.get("local_storage", {}), + session_storage=state.get("session_storage", {}), + metadata={"source": "manual_save"}, + ) + + return { + "success": True, + "data": {"session_id": session_id, "cookie_count": len(state.get("cookies", []))}, + } + + +@router.post("/v1/session/restore", tags=["Sessions"], summary="Restore a saved session") +async def restore_session(session_id: str = Body(...)) -> dict[str, Any]: + """Restore a session from disk into a browser context.""" + from sessions import load_session + + data = await load_session(session_id) + if not data: + raise NotFoundError(f"Saved session not found: {session_id}") + + success = await automator.restore_session_state( + session_id=session_id, + cookies=data.get("cookies", []), + ) + + return { + "success": success, + "data": { + "session_id": session_id, + "restored": success, + "cookie_count": len(data.get("cookies", [])), + "saved_at": data.get("saved_at", ""), + }, + } diff --git a/routers/share.py b/routers/share.py new file mode 100644 index 0000000..39693c2 --- /dev/null +++ b/routers/share.py @@ -0,0 +1,58 @@ +"""Pry โ€” Share router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import html +import logging +import uuid +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Body +from fastapi.responses import HTMLResponse + +from deps import scraper + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Share"]) + +@router.post("/v1/share", tags=["Share"], summary="Share scraped content via link") +async def share_scrape(data: dict[str, Any] = Body(...)) -> dict[str, Any]: + url = data.get("url", "") + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + sid = uuid.uuid4().hex[:8] + _shares[sid] = { + "url": url, + "title": result.get("title", url), + "content": result.get("content", ""), + "method": result.get("method"), + "ts": datetime.now(UTC).isoformat(), + } + return {"success": True, "data": {"share_id": sid, "url": f"/share/{sid}"}} + + +@router.get("/share/{share_id}", tags=["Share"], summary="View shared content") +async def view_share(share_id: str) -> HTMLResponse: + d = _shares.get(share_id) + if not d: + return HTMLResponse("

Not found

Share expired.

", 404) + st, sc, su = html.escape(d["title"]), html.escape(d["content"][:100000]), html.escape(d["url"]) + return HTMLResponse( + f'{st} โ€” Pry Share' + f'' + f"" + f'

๐Ÿ”ง {st}

{su} ยท {d["method"]} ยท {d["ts"][:10]}

' + f"
{sc}
" + ) + + +_shares: dict[str, dict[str, Any]] = {} diff --git a/routers/stats.py b/routers/stats.py new file mode 100644 index 0000000..f989d62 --- /dev/null +++ b/routers/stats.py @@ -0,0 +1,28 @@ +"""Pry โ€” Stats router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter + +from deps import automator, cache, ratelimiter + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Stats"]) + +@router.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats") +async def stats() -> dict[str, Any]: + return { + "cache": cache.stats(), + "rate_limiter": ratelimiter.get_stats(), + "sessions": automator.list_sessions(), + } diff --git a/routers/structure.py b/routers/structure.py new file mode 100644 index 0000000..6be264f --- /dev/null +++ b/routers/structure.py @@ -0,0 +1,110 @@ +"""Pry โ€” Structure router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError, NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Structure"]) + +@router.post( + "/v1/structure/check", + tags=["Structure"], + summary="Check if CSS selectors still match on a page", +) +async def structure_check( + url: str = Body(...), + selectors: list[dict[str, Any]] = Body(...), +) -> dict[str, Any]: + """Check if CSS selectors still match on a page. + + Use this to verify your scraper templates still work after + a site redesign. Returns per-selector match status. + + Selectors format: [{"name": "title", "selector": "h1", "type": "css"}] + """ + from structure_monitor import check_selectors + + result = await check_selectors(url, selectors) + return {"success": "error" not in result, "data": result} + + +@router.post( + "/v1/structure/monitor", tags=["Structure"], summary="Monitor page structure changes over time" +) +async def structure_monitor( + url: str = Body(...), + selectors: list[dict[str, Any]] = Body(...), + template_id: str = Body(""), +) -> dict[str, Any]: + """Monitor page structure changes and detect broken selectors. + + Compares current selector match status to previous checks. + Alerts when selectors stop matching (site redesign detected). + + Use this as a pre-scrape health check for your templates. + """ + from structure_monitor import monitor_page_structure + + result = await monitor_page_structure(url, selectors, template_id) + return {"success": "error" not in result, "data": result} + + +@router.post( + "/v1/structure/check-template", + tags=["Structure"], + summary="Verify a scraper template still works", +) +async def structure_check_template( + template_id: str = Body(...), url: str = Body("") +) -> dict[str, Any]: + """Check if a pre-built scraper template still works against a URL. + + Extracts the template's selectors and checks each one. + If selectors fail, the template needs updating. + """ + from structure_monitor import monitor_page_structure + from template_engine import get_template + + template = get_template(template_id) + if not template: + raise NotFoundError(f"Template not found: {template_id}") + + schema = template.get("schema", {}) + fields = schema.get("fields", []) + selectors = [ + { + "name": f.get("name", f"field_{i}"), + "selector": f.get("selector", ""), + "type": "css" if f.get("type") != "xpath" else "xpath", + } + for i, f in enumerate(fields) + if f.get("selector") + ] + + if not selectors: + raise InvalidRequestError("Template has no selectable fields") + + # If no URL provided, try the template's site URL + if not url: + site = template.get("site", "") + url = ( + f"https://www.{site}" + if site and not site.startswith("http") + else (site if site else "https://example.com") + ) + + result = await monitor_page_structure(url, selectors, template_id) + return {"success": "error" not in result, "data": result} diff --git a/routers/system.py b/routers/system.py new file mode 100644 index 0000000..781035d --- /dev/null +++ b/routers/system.py @@ -0,0 +1,44 @@ +"""Pry โ€” System router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Response +from fastapi.responses import JSONResponse + +from observability import get_metrics_output + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["System"]) + +@router.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False) +async def metrics() -> Response: + """Expose Prometheus metrics for scraping.""" + data, content_type = get_metrics_output() + return Response(content=data, media_type=content_type) + + +@router.get( + "/openapi.json", + tags=["System"], + summary="OpenAPI spec for AI agent integration", + include_in_schema=False, +) +async def get_openapi() -> JSONResponse: + """Get the OpenAPI specification for AI agent integration. + + Use this with ChatGPT GPT Actions, Claude MCP, or any AI agent + framework to let AI models scrape the web through Pry. + """ + from ai_plugin import get_openapi_spec + + spec = get_openapi_spec() + return JSONResponse(content=spec) diff --git a/routers/training.py b/routers/training.py new file mode 100644 index 0000000..38b829d --- /dev/null +++ b/routers/training.py @@ -0,0 +1,121 @@ +"""Pry โ€” Training router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from errors import NotFoundError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Training"]) + +@router.post( + "/v1/training/classify-license", + tags=["Training"], + summary="Classify content license for AI training", +) +async def classify_license_endpoint(text: str = Body(...)) -> dict[str, Any]: + """Classify the license of scraped content for AI training compliance. + + Returns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use), + tier (permissive/copyleft/restrictive/conditional), and confidence. + """ + from training_data import classify_license + + result = classify_license(text) + return {"success": True, "data": result} + + +@router.post("/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content") +async def clean_content( + text: str = Body(...), + strip_names: bool = Body(False), + strip_copyright: bool = Body(True), +) -> dict[str, Any]: + """Strip PII and copyright content for AI training clean room. + + Removes emails, phones, SSNs, credit cards, IPs, and (optionally) names. + Also strips copyright notices and near-verbatim copyright content. + """ + from training_data import strip_copyright_verbatim, strip_pii + + cleaned, pii_stats = strip_pii(text, preserve_names=not strip_names) + copyright_stats = {"blocks_removed": 0, "total_chars_removed": 0} + + if strip_copyright: + cleaned, copyright_stats = strip_copyright_verbatim(cleaned) + + return { + "success": True, + "data": { + "original_length": len(text), + "cleaned_length": len(cleaned), + "pii_removed": pii_stats, + "copyright_blocks_removed": copyright_stats["blocks_removed"], + "copyright_chars_removed": copyright_stats["total_chars_removed"], + "cleaned_content": cleaned[:5000] if len(cleaned) > 5000 else cleaned, + "truncated": len(cleaned) > 5000, + }, + } + + +@router.post( + "/v1/training/export", + tags=["Training"], + summary="Export a clean AI training dataset", +) +async def export_dataset( + records: list[dict[str, Any]] = Body(...), + output_format: str = Body("jsonl"), + clean_room: bool = Body(True), + strip_names: bool = Body(False), +) -> dict[str, Any]: + """Export scraped content as a clean AI training dataset. + + Each record should have: content, url, and optional metadata. + + Features: + - Per-record provenance tracking (source URL, timestamp, extraction method) + - PII stripping (email, phone, SSN, CC, IP) + - Copyright verbatim text removal + - License classification + - Compliance report generation + """ + from training_data import export_training_dataset + + result = export_training_dataset( + records=records, + format=output_format, # type: ignore + clean_room=clean_room, + strip_names=strip_names, + ) + return {"success": result["success"], "data": result} + + +@router.get( + "/v1/training/compliance/{dataset_id}", + tags=["Training"], + summary="Generate compliance report for a dataset", +) +async def compliance_report(dataset_id: str) -> dict[str, Any]: + """Generate a compliance report for an exported training dataset. + + Report includes: data provenance, PII/copyright removal stats, + license classification, and legal recommendations. + """ + from training_data import generate_compliance_report + + result = generate_compliance_report(dataset_id) + if "error" in result: + raise NotFoundError(result["error"]) + return {"success": True, "data": result} diff --git a/routers/transform.py b/routers/transform.py new file mode 100644 index 0000000..d9d72be --- /dev/null +++ b/routers/transform.py @@ -0,0 +1,68 @@ +"""Pry โ€” Transform router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import json +import logging +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import InvalidRequestError, ScrapeError +from pryextras import TransformEngine + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Transform"]) + +@router.post("/v1/transform", tags=["Transform"], summary="Transform data to multiple formats") +async def transform_data( + data: list[dict[str, Any]] = Body(...), format: str = "csv" +) -> dict[str, Any]: + te = TransformEngine() + if format == "csv": + result = te.to_csv(data) + elif format == "sql": + result = "\n".join(te.to_sql(row) for row in data) + elif format == "html": + result = te.to_html_table(data) + elif format == "markdown": + result = te.to_markdown_table(data) + else: + raise InvalidRequestError(f"Unknown format: {format}") + return {"success": True, "data": {"output": result, "format": format}} + + +@router.post("/v1/pipe", tags=["Transform"], summary="Scrape and transform via data pipeline") +async def data_pipeline(data: dict[str, Any] = Body(...)) -> dict[str, Any]: + url = data.get("url", "") + transform = data.get("transform", "json") + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Pipeline failed") + content = result.get("content", "") + title = result.get("title", url) + if transform == "sql": + from pryextras import TransformEngine + + te = TransformEngine() + output = te.to_sql({"url": url, "title": title, "content": content[:50000]}) + elif transform == "csv": + import csv + import io + + buf = io.StringIO() + w = csv.writer(buf) + w.writerow(["url", "title", "content"]) + w.writerow([url, title, content[:50000]]) + output = buf.getvalue() + else: + output = json.dumps({"url": url, "title": title, "content": content[:100000]}, indent=2) + return {"success": True, "data": {"url": url, "output": output, "format": transform}} diff --git a/routers/untagged.py b/routers/untagged.py new file mode 100644 index 0000000..0a74ed3 --- /dev/null +++ b/routers/untagged.py @@ -0,0 +1,30 @@ +"""Pry โ€” Untagged router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, WebSocket, WebSocketDisconnect + +from pryextras import streams + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Untagged"]) + +@router.websocket("/v1/stream/{job_id}") +async def stream_endpoint(websocket: WebSocket, job_id: str = "live") -> None: + await websocket.accept() + streams.register(job_id, websocket) + try: + while True: + data = await websocket.receive_text() + await streams.broadcast(job_id, {"echo": data}) + except WebSocketDisconnect: + streams.unregister(job_id, websocket) diff --git a/routers/vision.py b/routers/vision.py new file mode 100644 index 0000000..6a3fd01 --- /dev/null +++ b/routers/vision.py @@ -0,0 +1,221 @@ +"""Pry โ€” Vision router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import base64 +import logging +import os +from typing import Any + +import httpx +from fastapi import APIRouter, Body + +from client import get_client +from deps import automator +from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError +from settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Vision"]) + +@router.post("/v1/vision", tags=["Vision"], summary="Analyze an image with a free vision model") +async def vision( + question: str = Body("Describe what is visible in this image.", embed=True), + image: str | None = Body(None, embed=True), + url: str | None = Body(None, embed=True), + file_path: str | None = Body(None, embed=True), + model: str | None = Body(None, embed=True), + max_tokens: int = Body(800, embed=True), + session_id: str | None = Body(None, embed=True), + no_fallback: bool = Body(False, embed=True), +) -> dict[str, Any]: + """Analyze an image with a free vision model. + + Provide ONE of: image (base64 or data URI), url (auto-screenshot), + or file_path (local PNG/JPG). + + Auto-falls-back across 5 free OpenRouter vision models if the + requested one is rate-limited. + """ + try: + # 1. Resolve the image bytes + if url: + # Auto-screenshot via playwright + r = await automator.run_steps( + steps=[{"action": "navigate", "url": url}, {"action": "screenshot"}], + session_id=session_id, + ) + image_b64 = None + for step in r.get("steps", []): + if step.get("action") == "screenshot": + image_b64 = step.get("screenshot") + if not image_b64: + raise ScrapeError("screenshot returned empty") + elif file_path: + p = os.path.expanduser(file_path) + if not os.path.isfile(p): + raise InvalidRequestError(f"file not found: {file_path}") + with open(p, "rb") as f: + image_b64 = base64.b64encode(f.read()).decode("ascii") + elif image: + # Strip data URI prefix if present + image_b64 = image.split(",", 1)[1] if image.startswith("data:") else image + else: + raise InvalidRequestError("must provide one of: image, url, file_path") + + # 2. Build the fallback chain + if model: + models_to_try = [model] + if not no_fallback: + models_to_try += [m for m in VISION_MODELS_FALLBACK if m != model] + else: + models_to_try = list(VISION_MODELS_FALLBACK) + + # 3. Try each model until one succeeds + attempts = [] + for m in models_to_try: + try: + text, used, err = await _call_vision_api(image_b64, question, m, max_tokens) + if err: + attempts.append({"model": m, "error": err}) + continue + return { + "success": True, + "data": { + "answer": text, + "model_used": used, + "question": question, + "image_size_bytes": len(image_b64) * 3 // 4, + "attempts": attempts, + }, + } + except httpx.HTTPStatusError as e: + attempts.append( + { + "model": m, + "error": f"http {e.response.status_code}", + "body": e.response.text[:200], + } + ) + except OSError as e: + attempts.append({"model": m, "error": str(e)}) + + raise ExternalServiceError( + "all vision models failed", + details={"attempts": attempts}, + ) + except PryError: + raise + except OSError as e: + raise ExternalServiceError(str(e)) from e + + +@router.get( + "/v1/vision/models", tags=["Vision"], summary="List free vision-capable models on OpenRouter" +) +async def vision_models() -> dict[str, Any]: + """List free vision-capable models on OpenRouter.""" + try: + client = await get_client() + resp = await client.get( + "https://openrouter.ai/api/v1/models", headers={"User-Agent": "pry/3.0"}, timeout=15 + ) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as e: + raise ExternalServiceError(str(e)) from e + + free = [] + for m in data.get("data", []): + arch = m.get("architecture", {}) + if "image" not in arch.get("input_modalities", []): + continue + pricing = m.get("pricing", {}) + try: + pp = float(pricing.get("prompt", "1") or 1) + cp = float(pricing.get("completion", "1") or 1) + except (httpx.HTTPError, httpx.RequestError): + continue + if pp == 0 and cp == 0: + free.append( + { + "id": m["id"], + "name": m.get("name", ""), + "context": arch.get("context_length"), + "description": (m.get("description") or "")[:120], + } + ) + return {"success": True, "data": {"free_models": free, "count": len(free)}} + + +VISION_MODELS_FALLBACK = [ + "google/gemma-4-31b-it:free", + "google/gemma-4-26b-a4b-it:free", + "nvidia/nemotron-nano-12b-v2-vl:free", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free", + "openrouter/free", +] + + +async def _call_vision_api( + image_b64: str, question: str, model: str, max_tokens: int = 800 +) -> tuple[str | None, str | None, str | None]: + """Single async vision call. Returns (text, model_used) or raises.""" + key = _get_or_key() + if not key: + return None, None, "OPENROUTER_API_KEY not set" + + # Accept either data URI or raw base64 + data_uri = image_b64 if image_b64.startswith("data:") else f"data:image/png;base64,{image_b64}" + + payload = { + "model": model, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": question}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ], + } + ], + "max_tokens": max_tokens, + } + + client = await get_client() + resp = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + json=payload, + headers={ + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + "HTTP-Referer": "https://pry.local", + "X-Title": "pry-vision", + }, + timeout=60, + ) + resp.raise_for_status() + body = resp.json() + return body["choices"][0]["message"]["content"], model, None + + +def _get_or_key() -> str | None: + """Pull OPENROUTER_API_KEY from ~/.hermes/.env or env.""" + key = settings.openrouter_api_key + if key: + return key + env_path = os.path.expanduser("~/.hermes/.env") + if not os.path.isfile(env_path): + return None + with open(env_path) as f: + for line in f: + if line.startswith("OPENROUTER_API_KEY="): + return line.split("=", 1)[1].strip().strip('"').strip("'") + return None diff --git a/routers/webhooks.py b/routers/webhooks.py new file mode 100644 index 0000000..0b8acae --- /dev/null +++ b/routers/webhooks.py @@ -0,0 +1,46 @@ +"""Pry โ€” Webhooks router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Webhooks"]) + +@router.post( + "/v1/webhooks/test", + tags=["Webhooks"], + summary="Test webhook delivery", +) +async def test_webhook( + url: str = Body(...), + payload: dict[str, Any] | None = Body(None), +) -> dict[str, Any]: + """Test webhook delivery to a URL with HMAC signing.""" + from webhook_delivery import WebhookDelivery + + w = WebhookDelivery() + return await w.deliver(url, payload or {"test": True, "timestamp": "now"}, "test.event") + + +@router.get( + "/v1/webhooks/dead-letter", + tags=["Webhooks"], + summary="Get failed webhook deliveries", +) +async def get_dead_letter() -> dict[str, Any]: + """Get the dead letter queue of failed webhook deliveries.""" + from webhook_delivery import WebhookDelivery + + w = WebhookDelivery() + return {"success": True, "data": w.retry_dead_letter()} diff --git a/routers/x402.py b/routers/x402.py new file mode 100644 index 0000000..56cf02e --- /dev/null +++ b/routers/x402.py @@ -0,0 +1,153 @@ +"""Pry โ€” x402 router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from fastapi import APIRouter, Body + +from errors import InvalidRequestError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["x402"]) + +@router.get("/v1/x402/pricing", tags=["x402"], summary="Get x402 pricing for all paid operations") +async def x402_pricing() -> dict[str, Any]: + """Get the price list for pay-per-scrape operations.""" + from x402 import X402Handler + + return {"success": True, "data": X402Handler().get_stats()} + + +@router.post("/v1/x402/payment", tags=["x402"], summary="Create a x402 payment request") +async def x402_payment_request( + operation: str = Body(...), + metadata: dict[str, Any] | None = Body(None), +) -> dict[str, Any]: + """Create a x402 payment request for a paid operation. + + Returns payment details (wallet, amount, asset) for the client to pay. + """ + from x402 import create_payment_request + + req = create_payment_request(operation, metadata) + return {"success": True, "data": req} + + +@router.post("/v1/x402/verify", tags=["x402"], summary="Verify a x402 payment was settled") +async def x402_verify_payment( + payment_id: str = Body(...), + tx_hash: str = Body(...), + network: str = Body(""), + asset: str = Body(""), + amount_usd: float = Body(0.0), +) -> dict[str, Any]: + """Verify a x402 payment has been settled on-chain via facilitator router.""" + from x402 import X402Handler + + result = await X402Handler().verify_payment( + payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd + ) + return {"success": result["verified"], "data": result} + + +@router.post( + "/v1/x402/require-payment", tags=["x402"], summary="Generate a 402 Payment Required response" +) +async def x402_require_payment( + operation: str = Body(...), + metadata: dict[str, Any] | None = Body(None), +) -> dict[str, Any]: + """Generate a 402 Payment Required response for a paid endpoint.""" + from x402 import require_payment_legacy + + return require_payment_legacy(operation, metadata) + + +@router.post("/v1/x402/batch-payment", tags=["x402"], summary="Create a batch x402 payment") +async def x402_batch_payment(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Create a single x402 payment covering multiple operations. + + Returns a PaymentRequired body with the combined amount. After paying, + submit the tx to POST /v1/x402/batch-verify. + """ + from x402 import create_batch_payment + + operations = payload.get("operations", []) + if not isinstance(operations, list): + raise InvalidRequestError("operations must be a list") + result = await create_batch_payment(operations) + if "error" in result: + return {"success": False, "error": result["error"]} + return {"success": True, "data": result} + + +@router.post("/v1/x402/batch-verify", tags=["x402"], summary="Verify a batch x402 payment") +async def x402_batch_verify(payload: dict[str, Any] = Body(...)) -> dict[str, Any]: + """Verify the on-chain payment for a batch and mark it paid.""" + from x402 import verify_batch_payment + + batch_id = payload.get("batch_id", "") + tx_hash = payload.get("tx_hash", "") + network = payload.get("network", "") + asset = payload.get("asset", "") + if not batch_id or not tx_hash: + raise InvalidRequestError("batch_id and tx_hash are required") + result = await verify_batch_payment(batch_id, tx_hash, network=network, asset=asset) + return {"success": result["verified"], "data": result} + + +@router.post( + "/v1/x402/pay", + tags=["x402"], + summary="Process x402 payment and get access token", +) +async def x402_pay( + operation: str = Body(...), + tx_hash: str = Body(...), + payer_wallet: str = Body(...), + network: str = Body(""), + asset: str = Body(""), + amount_usd: float = Body(0.0), +) -> dict[str, Any]: + """Process an x402 payment and return an access token. + + Flow: + 1. User gets 402 from a paid endpoint + 2. User sends USDC to the wallet in the 402 response + 3. User calls this endpoint with the tx_hash + 4. Pry verifies the transaction through the facilitator router + 5. Returns access token (payment_id) to use in X-Payment-Hash header + """ + from x402 import X402Handler + + h = X402Handler() + payment_id = uuid.uuid4().hex[:12] + verify_result = await h.verify_payment( + payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd + ) + settlement = None + if verify_result.get("verified"): + settlement = await h.settle_payment( + payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd + ) + return { + "success": verify_result.get("verified", False), + "data": { + "payment_id": payment_id, + "tx_hash": tx_hash, + "verified": verify_result, + "settlement": settlement, + "payer_wallet": payer_wallet, + "use_in_header": {"X-Payment-ID": payment_id, "X-Payment-Hash": tx_hash}, + }, + } diff --git a/tests/test_api.py b/tests/test_api.py index 54cb3b4..595c343 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -20,7 +20,7 @@ def client() -> TestClient: def test_get_or_key_from_env(monkeypatch) -> None: monkeypatch.setattr("api.settings.openrouter_api_key", "sk-test-key-123") - from api import _get_or_key + from routers.vision import _get_or_key key = _get_or_key() assert key == "sk-test-key-123" @@ -29,14 +29,14 @@ def test_get_or_key_from_env(monkeypatch) -> None: def test_get_or_key_empty_when_not_set(monkeypatch) -> None: monkeypatch.setattr("api.settings.openrouter_api_key", "") monkeypatch.setattr("os.path.isfile", lambda p: False) - from api import _get_or_key + from routers.vision import _get_or_key key = _get_or_key() assert key is None def test_vision_models_fallback_list() -> None: - from api import VISION_MODELS_FALLBACK + from routers.vision import VISION_MODELS_FALLBACK assert len(VISION_MODELS_FALLBACK) >= 3 assert all("free" in m for m in VISION_MODELS_FALLBACK) -- 2.49.1