Scrape engine health and performance metrics
-| Endpoint | Method | Status |
|---|---|---|
| /v1/scrape | POST | โ Active |
| /v1/crawl | POST | โ Active |
| /v1/automate | POST | โ Active |
| /v1/batch | POST | โ Active |
| /v1/stream | WebSocket | โ Active |
| /v1/run | POST | โ Active |
| FlareSolverr | Proxy | โ Connected |
Pry v3.0.0 โ Generated {datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")}
-""" - return HTMLResponse(content=html) # โโ Win 1: AI Schema Suggestion โโ -@app.post("/v1/suggest", tags=["Analysis"], summary="AI-suggest schema fields for a URL") -async def suggest_schema(data: dict[str, Any] = Body(...)) -> dict[str, Any]: - url = data.get("url", "") - - result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30}) - content = result.get("content", "") - html = "" - try: - client = await get_client() - resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15) - html = resp.text - except httpx.HTTPError: - logger.warning("suggest_schema_fetch_failed", extra={"url": url}) - schema = {"_page_title": result.get("title", "")} - candidates: dict[str, str] = {} - for pattern, field in [ - (r'["\']((?:price|cost|amount)[^"\']*)["\']', "price"), - (r'["\'](product-title|product-name|item-name)[^"\']*["\']', "name"), - (r'["\'](?:product-image|main-image|featured-image)[^"\']*["\']', "image"), - (r'["\'](?:description|product-desc|item-desc)[^"\']*["\']', "description"), - (r'["\'](?:rating|stars|review-score)[^"\']*["\']', "rating"), - (r'["\'](?:stock|availability|status)[^"\']*["\']', "stock"), - ]: - found = re.findall(pattern, html.lower()) - if found: - candidates[field] = f'[class*="{found[0][:15]}"]' - if "name" not in candidates: - h1 = re.findall(r'Share expired.
", 404) - st, sc, su = html.escape(d["title"]), html.escape(d["content"][:100000]), html.escape(d["url"]) - return HTMLResponse( - f'{sc}"
- )
# โโ Compliance โโ
-@app.post("/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL")
-async def compliance_check(url: str = Body(...)) -> dict[str, Any]:
- """Run a full legal compliance check on a target URL.
-
- Analyzes:
- - robots.txt crawl permissions
- - Terms of Service classification (restrictive/permissive/moderate)
- - Jurisdiction detection (GDPR, CCPA, LGPD)
- - Sensitive data detection (PII, financial, health, contact)
-
- Returns a green/yellow/red risk score with recommendations.
- """
- from compliance import run_compliance_check
-
- result = await run_compliance_check(url)
- return {"success": True, "data": result}
-@app.get("/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation")
-async def compliance_docs() -> dict[str, Any]:
- """Get information about the compliance check endpoint."""
- return {
- "success": True,
- "data": {
- "description": "Legal compliance engine for web scraping targets",
- "risk_levels": {
- "green": "Low risk โ standard scraping practices",
- "yellow": "Moderate risk โ proceed with caution",
- "red": "High risk โ legal review required",
- },
- "factors_checked": [
- "robots.txt crawl permissions",
- "Terms of Service classification",
- "Jurisdiction detection (GDPR/CCPA/LGPD)",
- "Sensitive data categories (PII, financial, health, contact)",
- ],
- },
- }
# โโ GDPR Compliance Portal โโ
-@app.post("/v1/gdpr/consent", tags=["GDPR"], summary="Record user consent for data processing")
-async def record_consent_endpoint(
- user_id: str = Body(...),
- purpose: str = Body("data_collection"),
- consent_given: bool = Body(True),
- ip_address: str = Body(""),
- user_agent: str = Body(""),
-) -> dict[str, Any]:
- """Record a user's consent for data processing (GDPR Art. 7).
-
- Stores: user hash, purpose, timestamp, IP, user agent.
- Consent expires after 365 days.
- """
- from gdpr import record_consent
-
- result = await record_consent(user_id, purpose, consent_given, ip_address, user_agent)
- return {"success": result["success"], "data": result}
-@app.get("/v1/gdpr/consent/{user_id}", tags=["GDPR"], summary="Check user consent status")
-async def check_consent_endpoint(
- user_id: str,
- purpose: str = "data_collection",
-) -> dict[str, Any]:
- """Check if a user has given valid consent for data processing."""
- from gdpr import check_consent
-
- result = check_consent(user_id, purpose)
- return {"success": True, "data": result}
-@app.post("/v1/gdpr/consent/revoke", tags=["GDPR"], summary="Revoke user consent")
-async def revoke_consent_endpoint(
- user_id: str = Body(...),
- purpose: str = Body("data_collection"),
-) -> dict[str, Any]:
- """Revoke a user's consent for a processing purpose (GDPR Art. 7(3))."""
- from gdpr import revoke_consent
-
- result = revoke_consent(user_id, purpose)
- return {"success": True, "data": result}
-@app.post(
- "/v1/gdpr/deletion/request", tags=["GDPR"], summary="Request data deletion (right to erasure)"
-)
-async def request_deletion_endpoint(
- user_id: str = Body(...),
- reason: str = Body("user_request"),
-) -> dict[str, Any]:
- """Request deletion of all data associated with a user (GDPR Art. 17).
-
- Creates a deletion request that can be executed immediately or
- after a holding period.
- """
- from gdpr import request_deletion
-
- result = await request_deletion(user_id, reason)
- return {"success": "error" not in result, "data": result}
-@app.post("/v1/gdpr/deletion/execute", tags=["GDPR"], summary="Execute data deletion request")
-async def execute_deletion_endpoint(
- request_id: str = Body(...),
-) -> dict[str, Any]:
- """Execute a pending deletion request, removing all user data."""
- from gdpr import execute_deletion
-
- result = await execute_deletion(request_id)
- if "error" in result:
- raise NotFoundError(result["error"])
- return {"success": True, "data": result}
-@app.get("/v1/gdpr/retention", tags=["GDPR"], summary="Get data retention policy")
-async def get_retention_policy_endpoint() -> dict[str, Any]:
- """Get the current data retention policy."""
- from gdpr import get_retention_policy
-
- return {"success": True, "data": get_retention_policy()}
-@app.post(
- "/v1/gdpr/retention/apply",
- tags=["GDPR"],
- summary="Apply retention policy to remove expired data",
-)
-async def apply_retention() -> dict[str, Any]:
- """Apply the retention policy, removing all expired data."""
- from gdpr import apply_retention_policy
-
- result = await apply_retention_policy()
- return {"success": True, "data": result}
-@app.get("/v1/gdpr/audit", tags=["GDPR"], summary="Get compliance audit log")
-async def get_audit_log_endpoint(days_back: int = 7) -> dict[str, Any]:
- """Get the compliance audit log for the specified period."""
- from gdpr import get_audit_log
-
- events = get_audit_log(days_back)
- return {"success": True, "data": {"events": events, "total": len(events)}}
# โโ AI Training Data Pipeline โโ
-@app.post(
- "/v1/training/classify-license",
- tags=["Training"],
- summary="Classify content license for AI training",
-)
-async def classify_license_endpoint(text: str = Body(...)) -> dict[str, Any]:
- """Classify the license of scraped content for AI training compliance.
-
- Returns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use),
- tier (permissive/copyleft/restrictive/conditional), and confidence.
- """
- from training_data import classify_license
-
- result = classify_license(text)
- return {"success": True, "data": result}
-@app.post("/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content")
-async def clean_content(
- text: str = Body(...),
- strip_names: bool = Body(False),
- strip_copyright: bool = Body(True),
-) -> dict[str, Any]:
- """Strip PII and copyright content for AI training clean room.
-
- Removes emails, phones, SSNs, credit cards, IPs, and (optionally) names.
- Also strips copyright notices and near-verbatim copyright content.
- """
- from training_data import strip_copyright_verbatim, strip_pii
-
- cleaned, pii_stats = strip_pii(text, preserve_names=not strip_names)
- copyright_stats = {"blocks_removed": 0, "total_chars_removed": 0}
-
- if strip_copyright:
- cleaned, copyright_stats = strip_copyright_verbatim(cleaned)
-
- return {
- "success": True,
- "data": {
- "original_length": len(text),
- "cleaned_length": len(cleaned),
- "pii_removed": pii_stats,
- "copyright_blocks_removed": copyright_stats["blocks_removed"],
- "copyright_chars_removed": copyright_stats["total_chars_removed"],
- "cleaned_content": cleaned[:5000] if len(cleaned) > 5000 else cleaned,
- "truncated": len(cleaned) > 5000,
- },
- }
-@app.post(
- "/v1/training/export",
- tags=["Training"],
- summary="Export a clean AI training dataset",
-)
-async def export_dataset(
- records: list[dict[str, Any]] = Body(...),
- output_format: str = Body("jsonl"),
- clean_room: bool = Body(True),
- strip_names: bool = Body(False),
-) -> dict[str, Any]:
- """Export scraped content as a clean AI training dataset.
-
- Each record should have: content, url, and optional metadata.
-
- Features:
- - Per-record provenance tracking (source URL, timestamp, extraction method)
- - PII stripping (email, phone, SSN, CC, IP)
- - Copyright verbatim text removal
- - License classification
- - Compliance report generation
- """
- from training_data import export_training_dataset
-
- result = export_training_dataset(
- records=records,
- format=output_format, # type: ignore
- clean_room=clean_room,
- strip_names=strip_names,
- )
- return {"success": result["success"], "data": result}
-@app.get(
- "/v1/training/compliance/{dataset_id}",
- tags=["Training"],
- summary="Generate compliance report for a dataset",
-)
-async def compliance_report(dataset_id: str) -> dict[str, Any]:
- """Generate a compliance report for an exported training dataset.
-
- Report includes: data provenance, PII/copyright removal stats,
- license classification, and legal recommendations.
- """
- from training_data import generate_compliance_report
-
- result = generate_compliance_report(dataset_id)
- if "error" in result:
- raise NotFoundError(result["error"])
- return {"success": True, "data": result}
# โโ Monitoring (cron-based content change detection with AI judging) โโ
-@app.post("/v1/monitor", tags=["Monitoring"], summary="Create a scheduled content monitor")
-async def create_monitor_endpoint(
- name: str = Body(...),
- url: str = Body(...),
- schedule_cron: str = Body("0 */6 * * *"),
- goal: str = Body(""),
- webhook: str = Body(""),
- use_llm: bool = Body(False),
-) -> dict[str, Any]:
- """Create a scheduled monitor that tracks content changes.
-
- Args:
- name: Human-readable monitor name
- url: Target URL to monitor
- schedule_cron: Cron expression (default: every 6 hours)
- goal: Natural language goal for meaningful-change detection
- webhook: URL to notify on meaningful changes
- use_llm: Use LLM for change judging (slower but smarter)
- """
- from monitor import create_monitor
-
- monitor = await create_monitor(name, url, schedule_cron, goal, webhook, use_llm)
- return {"success": True, "data": monitor}
-@app.post("/v1/monitor/run", tags=["Monitoring"], summary="Run a single monitor check")
-async def run_monitor_endpoint(
- monitor_id: str = Body(...),
-) -> dict[str, Any]:
- """Execute a monitor check immediately (outside of schedule)."""
- from monitor import run_monitor
-
- result = await run_monitor(monitor_id)
- if "error" in result:
- raise NotFoundError(result["error"])
- return {"success": True, "data": result}
-@app.get("/v1/monitors", tags=["Monitoring"], summary="List all monitors")
-async def list_monitors_endpoint() -> dict[str, Any]:
- """List all registered monitors."""
- from monitor import list_monitors
-
- monitors = await list_monitors()
- return {"success": True, "data": {"monitors": monitors, "total": len(monitors)}}
-@app.delete("/v1/monitor/{monitor_id}", tags=["Monitoring"], summary="Delete a monitor")
-async def delete_monitor_endpoint(monitor_id: str) -> dict[str, Any]:
- """Delete a monitor and its snapshots."""
- from monitor import delete_monitor
-
- success = await delete_monitor(monitor_id)
- if not success:
- raise NotFoundError(f"Monitor not found: {monitor_id}")
- return {"success": True, "data": {"monitor_id": monitor_id, "deleted": True}}
-@app.post("/v1/monitor/check", tags=["Monitoring"], summary="Run all due monitors")
-async def run_due_monitors() -> dict[str, Any]:
- """Check all monitors and run any that are due based on their cron schedule."""
- import croniter
-
- from monitor import list_monitors, run_monitor
-
- monitors = await list_monitors()
- now = datetime.now(UTC)
- results = []
- for m in monitors:
- last_run = m.get("last_run_at")
- last_dt = datetime.fromisoformat(last_run) if last_run else datetime.min.replace(tzinfo=UTC)
-
- try:
- cron = croniter.croniter(m["schedule_cron"], last_dt)
- next_run = cron.get_next(datetime)
- if next_run <= now:
- result = await run_monitor(m["id"])
- results.append(result)
- except Exception as e: # noqa: BLE001
- logger.warning(
- "monitor_schedule_check_failed", extra={"monitor_id": m["id"], "error": str(e)}
- )
-
- return {
- "success": True,
- "data": {
- "total_monitors": len(monitors),
- "ran_count": len(results),
- "results": results,
- },
- }
# โโ Structure Monitor โโ
-@app.post(
- "/v1/structure/check",
- tags=["Structure"],
- summary="Check if CSS selectors still match on a page",
-)
-async def structure_check(
- url: str = Body(...),
- selectors: list[dict[str, Any]] = Body(...),
-) -> dict[str, Any]:
- """Check if CSS selectors still match on a page.
-
- Use this to verify your scraper templates still work after
- a site redesign. Returns per-selector match status.
-
- Selectors format: [{"name": "title", "selector": "h1", "type": "css"}]
- """
- from structure_monitor import check_selectors
-
- result = await check_selectors(url, selectors)
- return {"success": "error" not in result, "data": result}
-@app.post(
- "/v1/structure/monitor", tags=["Structure"], summary="Monitor page structure changes over time"
-)
-async def structure_monitor(
- url: str = Body(...),
- selectors: list[dict[str, Any]] = Body(...),
- template_id: str = Body(""),
-) -> dict[str, Any]:
- """Monitor page structure changes and detect broken selectors.
-
- Compares current selector match status to previous checks.
- Alerts when selectors stop matching (site redesign detected).
-
- Use this as a pre-scrape health check for your templates.
- """
- from structure_monitor import monitor_page_structure
-
- result = await monitor_page_structure(url, selectors, template_id)
- return {"success": "error" not in result, "data": result}
-@app.post(
- "/v1/structure/check-template",
- tags=["Structure"],
- summary="Verify a scraper template still works",
-)
-async def structure_check_template(
- template_id: str = Body(...), url: str = Body("")
-) -> dict[str, Any]:
- """Check if a pre-built scraper template still works against a URL.
-
- Extracts the template's selectors and checks each one.
- If selectors fail, the template needs updating.
- """
- from structure_monitor import monitor_page_structure
- from template_engine import get_template
-
- template = get_template(template_id)
- if not template:
- raise NotFoundError(f"Template not found: {template_id}")
-
- schema = template.get("schema", {})
- fields = schema.get("fields", [])
- selectors = [
- {
- "name": f.get("name", f"field_{i}"),
- "selector": f.get("selector", ""),
- "type": "css" if f.get("type") != "xpath" else "xpath",
- }
- for i, f in enumerate(fields)
- if f.get("selector")
- ]
-
- if not selectors:
- raise InvalidRequestError("Template has no selectable fields")
-
- # If no URL provided, try the template's site URL
- if not url:
- site = template.get("site", "")
- url = (
- f"https://www.{site}"
- if site and not site.startswith("http")
- else (site if site else "https://example.com")
- )
-
- result = await monitor_page_structure(url, selectors, template_id)
- return {"success": "error" not in result, "data": result}
# โโ Intelligence (Competitive Intelligence Engine) โโ
-@app.post("/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot")
-async def record_intel_snapshot(
- competitor_id: str = Body(...),
- competitor_name: str = Body(...),
- url: str = Body(...),
- fields: dict[str, Any] = Body(...),
-) -> dict[str, Any]:
- """Record a data snapshot for a competitor.
-
- Snapshots are stored with timestamps and used for trend analysis,
- anomaly detection, and report generation.
- """
- from intelligence import record_snapshot
-
- result = record_snapshot(competitor_id, competitor_name, url, fields)
- return {"success": True, "data": result}
-@app.get(
- "/v1/intel/snapshots/{competitor_id}", tags=["Intelligence"], summary="Get competitor snapshots"
-)
-async def get_intel_snapshots(
- competitor_id: str,
- limit: int = 50,
- since_hours: int | None = None,
-) -> dict[str, Any]:
- """Get historical snapshots for a competitor."""
- from intelligence import get_snapshots
-
- snapshots = get_snapshots(competitor_id, limit, since_hours)
- return {
- "success": True,
- "data": {"competitor_id": competitor_id, "snapshots": snapshots, "count": len(snapshots)},
- }
-@app.post(
- "/v1/intel/analyze", tags=["Intelligence"], summary="Analyze competitor field for anomalies"
-)
-async def analyze_field(
- competitor_id: str = Body(...),
- field: str = Body("price"),
-) -> dict[str, Any]:
- """Analyze a specific field across a competitor's snapshots for anomalies.
-
- Returns statistics, z-score analysis, and anomaly detection.
- """
- from intelligence import compute_field_statistics, detect_anomalies_numeric, get_snapshots
-
- snapshots = get_snapshots(competitor_id, limit=100)
- stats = compute_field_statistics(snapshots, field)
-
- if (
- stats.get("has_history")
- and stats.get("latest") is not None
- and stats.get("previous") is not None
- ):
- numeric_vals = [
- s.get("fields", {}).get(field)
- for s in snapshots
- if isinstance(s.get("fields", {}).get(field), (int, float))
- ]
- if len(numeric_vals) >= 3:
- anomaly = detect_anomalies_numeric(numeric_vals[-1], numeric_vals[:-1])
- else:
- anomaly = {"anomaly": False, "reason": "Insufficient numeric history"}
- else:
- anomaly = {"anomaly": False, "reason": "Insufficient data"}
-
- return {"success": True, "data": {"field": field, "statistics": stats, "anomaly": anomaly}}
-@app.post(
- "/v1/intel/report", tags=["Intelligence"], summary="Generate a competitive intelligence report"
-)
-async def generate_report(
- competitors: list[dict[str, Any]] = Body(...),
- days_back: int = Body(7),
-) -> dict[str, Any]:
- """Generate a competitive intelligence report for tracked competitors.
-
- Analyzes changes over the specified period and returns a structured report
- with a natural-language summary.
- """
- from intelligence import generate_weekly_report
-
- report = generate_weekly_report(competitors, days_back)
- return {"success": True, "data": report}
# โโ Quality โโ
-@app.post(
- "/v1/quality/check", tags=["Quality"], summary="Run data quality check on extraction results"
-)
-async def quality_check(
- url: str = Body(...),
- data: dict[str, Any] = Body(...),
- schema: dict[str, Any] | None = Body(None),
- expected_types: dict[str, str] | None = Body(None),
-) -> dict[str, Any]:
- """Run a full data quality check on extraction results.
-
- Metrics:
- - Completeness: what % of expected fields have values
- - Schema adherence: field types match expectations
- - Freshness: how old is the data
- - Anomaly detection: what changed since last extraction
-
- Use this to validate data BEFORE sending to downstream systems.
- """
- from quality import run_quality_check
-
- # Convert string type names to actual types
- type_map: dict[str, type] = {
- "str": str,
- "int": int,
- "float": float,
- "bool": bool,
- "list": list,
- "dict": dict,
- "None": type(None),
- }
- resolved_types: dict[str, type] | None = None
- if expected_types:
- resolved_types = {}
- for field, type_name in expected_types.items():
- resolved_types[field] = type_map.get(type_name, str)
-
- result = await run_quality_check(
- url=url,
- data=data,
- schema=schema,
- expected_types=resolved_types,
- )
- return {"success": True, "data": result}
-@app.get(
- "/v1/quality/stats", tags=["Quality"], summary="Get quality statistics for all checked URLs"
-)
-async def quality_stats() -> dict[str, Any]:
- """Get aggregate quality statistics across all checked URLs."""
- from quality import QUALITY_DIR
-
- stats: list[dict[str, Any]] = []
- for path in sorted(QUALITY_DIR.glob("*.json"), key=os.path.getmtime, reverse=True)[:50]:
- try:
- data = json.loads(path.read_text())
- stats.append(
- {
- "url": data.get("url", ""),
- "checked_at": data.get("checked_at", ""),
- "size_bytes": path.stat().st_size,
- }
- )
- except (json.JSONDecodeError, OSError):
- continue
-
- return {"success": True, "data": {"total_checked": len(stats), "recent": stats}}
# โโ Reconciliation โโ
-@app.post(
- "/v1/reconcile",
- tags=["Reconciliation"],
- summary="Reconcile records from multiple sources into unified entities",
-)
-async def reconcile_endpoint(
- records: list[dict[str, Any]] = Body(...),
- vertical: str = Body("product"),
- threshold: float = Body(0.7),
-) -> dict[str, Any]:
- """Reconcile records from multiple sources into matched entities.
-
- Matches records across sources using identity field similarity,
- normalizes to a unified vertical schema, and returns entity groups
- with confidence scores.
-
- Verticals: product, job, real_estate, review
- """
- from reconciliation import VERTICAL_SCHEMAS, reconcile
-
- if vertical not in VERTICAL_SCHEMAS:
- raise InvalidRequestError(
- f"Unknown vertical: {vertical}. Supported: {list(VERTICAL_SCHEMAS.keys())}"
- )
-
- result = await reconcile(records, vertical, threshold)
- return {"success": True, "data": result}
-@app.get(
- "/v1/reconcile/schemas",
- tags=["Reconciliation"],
- summary="List supported reconciliation schemas",
-)
-async def list_schemas() -> dict[str, Any]:
- """List all supported vertical schemas for entity reconciliation."""
- from reconciliation import VERTICAL_SCHEMAS
-
- schemas = {}
- for key, schema in VERTICAL_SCHEMAS.items():
- schemas[key] = {
- "name": schema["name"],
- "fields": {
- k: {fk: fv for fk, fv in v.items() if fk != "type"}
- for k, v in schema["fields"].items()
- },
- "identity_fields": schema["identity_fields"],
- }
-
- return {"success": True, "data": {"schemas": schemas, "total": len(schemas)}}
# โโ Auth โโ
@@ -2809,1440 +682,251 @@ app.include_router(auth_router)
app.include_router(health_router)
app.include_router(scraping_router)
app.include_router(templates_router)
+app.include_router(ai_router)
+app.include_router(advanced_router)
+app.include_router(agency_router)
+app.include_router(alerts_router)
+app.include_router(analysis_router)
+app.include_router(automation_router)
+app.include_router(batch_router)
+app.include_router(crm_router)
+app.include_router(circuit_breaker_router)
+app.include_router(commerce_router)
+app.include_router(compliance_router)
+app.include_router(config_router)
+app.include_router(costing_router)
+app.include_router(dashboard_router)
+app.include_router(email_router)
+app.include_router(enrichment_router)
+app.include_router(execute_router)
+app.include_router(export_router)
+app.include_router(extraction_router)
+app.include_router(freshness_router)
+app.include_router(gdpr_router)
+app.include_router(integrations_router)
+app.include_router(intelligence_router)
+app.include_router(jobs_router)
+app.include_router(marketplace_router)
+app.include_router(monitoring_router)
+app.include_router(parsing_router)
+app.include_router(pipeline_router)
+app.include_router(pipelines_router)
+app.include_router(proxy_router)
+app.include_router(quality_router)
+app.include_router(reconciliation_router)
+app.include_router(recorder_router)
+app.include_router(referrals_router)
+app.include_router(reports_router)
+app.include_router(review_router)
+app.include_router(seo_router)
+app.include_router(scraping_api_router)
+app.include_router(sessions_router)
+app.include_router(share_router)
+app.include_router(stats_router)
+app.include_router(structure_router)
+app.include_router(system_router)
+app.include_router(training_router)
+app.include_router(transform_router)
+app.include_router(untagged_router)
+app.include_router(vision_router)
+app.include_router(webhooks_router)
+app.include_router(x402_router)
+
+
# โโ Review โโ
-@app.post("/v1/review/submit", tags=["Review"], summary="Submit extracted data for human review")
-async def review_submit(
- data: dict[str, Any] = Body(...),
- url: str = Body(...),
- schema_name: str | None = Body(None),
- confidence_score: float = Body(0.0),
- flagged_fields: list[dict[str, Any]] | None = Body(None),
-) -> dict[str, Any]:
- """Submit extracted data for human review before delivery.
-
- Use this when extraction confidence is low or anomalies were detected.
- Data will be held in the review queue until approved or rejected.
- """
- from review import submit_for_review
-
- result = await submit_for_review(
- data=data,
- extraction_url=url,
- schema_name=schema_name,
- confidence_score=confidence_score,
- flagged_fields=flagged_fields,
- )
- return {"success": True, "data": result}
-@app.post("/v1/review/{review_id}/approve", tags=["Review"], summary="Approve a review item")
-async def review_approve(
- review_id: str,
- reviewer: str = Body("api"),
- notes: str = Body(""),
-) -> dict[str, Any]:
- """Approve a review item, allowing data to proceed to delivery."""
- from review import approve_review
-
- result = await approve_review(review_id, reviewer, notes)
- if "error" in result:
- raise NotFoundError(result["error"])
- return {"success": True, "data": result}
-@app.post("/v1/review/{review_id}/reject", tags=["Review"], summary="Reject a review item")
-async def review_reject(
- review_id: str,
- reviewer: str = Body("api"),
- notes: str = Body(""),
-) -> dict[str, Any]:
- """Reject a review item, blocking data delivery."""
- from review import reject_review
-
- result = await reject_review(review_id, reviewer, notes)
- if "error" in result:
- raise NotFoundError(result["error"])
- return {"success": True, "data": result}
-@app.get("/v1/reviews", tags=["Review"], summary="List reviews in the queue")
-async def list_reviews(status: str | None = None) -> dict[str, Any]:
- """List reviews, optionally filtered by status (pending/approved/rejected)."""
- from review import get_review_queue
-
- reviews = get_review_queue(status)
- return {"success": True, "data": {"reviews": reviews, "total": len(reviews)}}
-@app.get("/v1/review/{review_id}", tags=["Review"], summary="Get review details")
-async def get_review(review_id: str) -> dict[str, Any]:
- """Get full details of a review item including the data payload."""
- from review import get_review_detail
-
- result = get_review_detail(review_id)
- if not result:
- raise NotFoundError(f"Review not found: {review_id}")
- return {"success": True, "data": result}
-@app.post(
- "/v1/extract-with-review",
- tags=["Review"],
- summary="Extract with automatic human review routing",
-)
-async def extract_with_review(
- url: str = Body(...),
- schema: dict[str, Any] | None = Body(None),
- expected_types: dict[str, str] | None = Body(None),
- slack_webhook: str = Body(""),
- auto_approve_threshold: float = Body(0.8),
- auto_reject_threshold: float = Body(0.2),
-) -> dict[str, Any]:
- """Extract data with automatic quality check and human review routing.
-
- High-confidence results are auto-approved.
- Low-confidence results are auto-rejected.
- Medium-confidence results go to the human review queue with Slack notification.
- """
- from quality import run_quality_check
- from review import auto_review_threshold
-
- # Scrape
- scrape_result = await scraper.scrape(url, {"bypass_cloudflare": True})
- if scrape_result.get("status") != "ok":
- raise ScrapeError(scrape_result.get("error") or "Scrape failed")
-
- data = scrape_result
-
- # Quality check
- quality = await run_quality_check(
- url=url,
- data=data,
- schema=schema,
- expected_types=None,
- )
-
- # Auto-route
- decision = await auto_review_threshold(
- data=data,
- extraction_url=url,
- quality_result=quality,
- slack_webhook=slack_webhook,
- auto_approve_threshold=auto_approve_threshold,
- auto_reject_threshold=auto_reject_threshold,
- )
-
- return {
- "success": True,
- "data": {
- "decision": decision,
- "quality": {k: v for k, v in quality.items() if k != "url"},
- },
- }
# โโ Commerce Sync โโ
-@app.post("/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify")
-async def sync_commerce(
- platform: str = Body(...),
- products: list[dict[str, Any]] = Body(...),
- credentials: dict[str, Any] = Body(...),
-) -> dict[str, Any]:
- """Sync scraped products to your e-commerce platform.
-
- Platforms:
- - woocommerce: credentials = {"wp_url": "...", "consumer_key": "...", "consumer_secret": "..."}
- - shopify: credentials = {"shop_url": "...", "access_token": "..."}
-
- Products are imported as drafts for review before publishing.
- """
- from commerce_sync import sync_to_shopify, sync_to_woocommerce
-
- if platform == "woocommerce":
- result = await sync_to_woocommerce(
- products=products,
- wp_url=credentials.get("wp_url", ""),
- consumer_key=credentials.get("consumer_key", ""),
- consumer_secret=credentials.get("consumer_secret", ""),
- category_id=credentials.get("category_id", 0),
- status=credentials.get("status", "draft"),
- )
- elif platform == "shopify":
- result = await sync_to_shopify(
- products=products,
- shop_url=credentials.get("shop_url", ""),
- access_token=credentials.get("access_token", ""),
- )
- else:
- raise InvalidRequestError(
- f"Unsupported platform: {platform}. Supported: woocommerce, shopify"
- )
-
- return {"success": result["success"], "data": result}
-@app.get("/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms")
-async def list_commerce_platforms() -> dict[str, Any]:
- """List supported e-commerce platforms and their credential requirements."""
- return {
- "success": True,
- "data": {
- "platforms": [
- {
- "id": "woocommerce",
- "name": "WooCommerce",
- "credential_fields": [
- "wp_url",
- "consumer_key",
- "consumer_secret",
- "category_id",
- ],
- "product_fields": ["name", "price", "description", "image_url"],
- },
- {
- "id": "shopify",
- "name": "Shopify",
- "credential_fields": ["shop_url", "access_token"],
- "product_fields": ["name", "price", "description", "image_url"],
- },
- ]
- },
- }
# โโ CRM Sync โโ
-@app.post("/v1/crm/sync", tags=["CRM"], summary="Sync scraped data to CRM")
-async def sync_crm(
- platform: str = Body(...),
- object_type: str = Body("lead"),
- objects: list[dict[str, Any]] = Body(...),
- credentials: dict[str, Any] = Body(...),
-) -> dict[str, Any]:
- """Sync scraped data to your CRM.
-
- Platforms:
- - salesforce: credentials={"instance_url": "...", "access_token": "..."}
- - hubspot: credentials={"api_key": "..."}
- - pipedrive: credentials={"api_token": "...", "domain": "..."}
- - close: credentials={"api_key": "..."}
-
- Object types vary by platform:
- - Salesforce: Lead, Contact, Account, Opportunity
- - HubSpot: contacts, companies, deals
- - Pipedrive: person, organization, deal, lead
- - Close: lead, contact
- """
- from crm_sync import sync_to_close, sync_to_hubspot, sync_to_pipedrive, sync_to_salesforce
-
- platform_map = {
- "salesforce": (sync_to_salesforce, ("instance_url", "access_token")),
- "hubspot": (sync_to_hubspot, ("api_key",)),
- "pipedrive": (sync_to_pipedrive, ("api_token",)),
- "close": (sync_to_close, ("api_key",)),
- }
-
- if platform not in platform_map:
- raise InvalidRequestError(
- f"Unsupported platform: {platform}. Supported: {list(platform_map.keys())}"
- )
-
- handler, required_fields = platform_map[platform]
- for field in required_fields:
- if not credentials.get(field):
- raise InvalidRequestError(f"Missing required credential: {field}")
-
- if platform == "salesforce":
- result = await handler(
- objects, object_type, credentials["instance_url"], credentials["access_token"]
- )
- elif platform == "hubspot":
- result = await handler(objects, object_type, credentials["api_key"])
- elif platform == "pipedrive":
- result = await handler(
- objects, object_type, credentials["api_token"], credentials.get("domain", "")
- )
- elif platform == "close":
- result = await handler(objects, object_type, credentials["api_key"])
- else:
- raise InvalidRequestError(f"Platform {platform} not handled")
-
- return {"success": result["success"], "data": result}
-@app.get("/v1/crm/platforms", tags=["CRM"], summary="List supported CRM platforms")
-async def list_crm_platforms() -> dict[str, Any]:
- """List supported CRM platforms and their credential requirements."""
- return {
- "success": True,
- "data": {
- "platforms": [
- {
- "id": "salesforce",
- "name": "Salesforce",
- "objects": ["Lead", "Contact", "Account", "Opportunity"],
- "credential_fields": ["instance_url", "access_token"],
- },
- {
- "id": "hubspot",
- "name": "HubSpot",
- "objects": ["contacts", "companies", "deals"],
- "credential_fields": ["api_key"],
- },
- {
- "id": "pipedrive",
- "name": "Pipedrive",
- "objects": ["person", "organization", "deal", "lead"],
- "credential_fields": ["api_token", "domain"],
- },
- {
- "id": "close",
- "name": "Close.com",
- "objects": ["lead", "contact"],
- "credential_fields": ["api_key"],
- },
- ]
- },
- }
# โโ Pipeline Builder Endpoints โโ
-@app.get(
- "/v1/pipelines/steps", tags=["Pipelines"], summary="List all available pipeline step types"
-)
-async def list_step_types() -> dict[str, Any]:
- """List all available step types for the visual pipeline builder.
-
- Each step type includes: name, icon, description, required inputs with types,
- and expected outputs. A UI renders these as drag-and-drop blocks.
- """
- from pipelines import STEP_TYPES
-
- return {"success": True, "data": {"step_types": STEP_TYPES, "total": len(STEP_TYPES)}}
-@app.post("/v1/pipelines/validate", tags=["Pipelines"], summary="Validate a pipeline definition")
-async def validate_pipeline_endpoint(pipeline: dict[str, Any] = Body(...)) -> dict[str, Any]:
- """Validate a pipeline definition for correctness."""
- from pipelines import validate_pipeline
-
- errors = validate_pipeline(pipeline)
- return {"success": len(errors) == 0, "data": {"valid": len(errors) == 0, "errors": errors}}
-@app.post("/v1/pipelines/run", tags=["Pipelines"], summary="Execute a pipeline")
-async def run_pipeline_endpoint(
- pipeline: dict[str, Any] = Body(...),
- context: dict[str, Any] | None = Body(None),
-) -> dict[str, Any]:
- """Execute a pipeline definition.
-
- Steps run sequentially. Each step's output is available as
- {{step_id.output_key}} in subsequent step templates.
-
- Example pipeline:
- {
- "name": "Monitor Competitor Pricing",
- "steps": [
- {"id": "scrape_amazon", "type": "scrape", "inputs": {"url": "https://amazon.com/product"}},
- {"id": "extract", "type": "extract_css", "inputs": {"url": "{{scrape_amazon.url}}", "schema": {...}}},
- {"id": "quality", "type": "quality_check", "inputs": {"url": "{{scrape_amazon.url}}", "data": "{{extract.items}}"}},
- {"id": "notify", "type": "send_slack", "inputs": {"webhook_url": "https://hooks.slack.com/...", "message": "{{quality.quality_score}}"}},
- ]
- }
- """
- from pipelines import run_pipeline, validate_pipeline
-
- errors = validate_pipeline(pipeline)
- if errors:
- raise InvalidRequestError(f"Pipeline validation failed: {'; '.join(errors)}")
- result = await run_pipeline(pipeline, context)
- return {"success": not result["failed"], "data": result}
-@app.post("/v1/pipelines/save", tags=["Pipelines"], summary="Save a pipeline definition")
-async def save_pipeline_endpoint(pipeline: dict[str, Any] = Body(...)) -> dict[str, Any]:
- """Save a pipeline definition for later use."""
- from pipelines import save_pipeline
-
- result = save_pipeline(pipeline)
- return {"success": result["success"], "data": result}
-@app.get("/v1/pipelines", tags=["Pipelines"], summary="List saved pipelines")
-async def list_pipelines_endpoint() -> dict[str, Any]:
- """List all saved pipeline definitions."""
- from pipelines import list_pipelines
-
- pipelines = list_pipelines()
- return {"success": True, "data": {"pipelines": pipelines, "total": len(pipelines)}}
-@app.get("/v1/pipelines/{pipeline_id}", tags=["Pipelines"], summary="Get a saved pipeline")
-async def get_pipeline_endpoint(pipeline_id: str) -> dict[str, Any]:
- """Get a saved pipeline definition by ID."""
- from pipelines import get_pipeline
-
- result = get_pipeline(pipeline_id)
- if not result:
- raise NotFoundError(f"Pipeline not found: {pipeline_id}")
- return {"success": True, "data": result}
-@app.delete("/v1/pipelines/{pipeline_id}", tags=["Pipelines"], summary="Delete a saved pipeline")
-async def delete_pipeline_endpoint(pipeline_id: str) -> dict[str, Any]:
- """Delete a saved pipeline definition."""
- from pipelines import delete_pipeline
-
- success = delete_pipeline(pipeline_id)
- if not success:
- raise NotFoundError(f"Pipeline not found: {pipeline_id}")
- return {"success": True, "data": {"deleted": True}}
# โโ Agency โโ
-@app.post("/v1/agency/create", tags=["Agency"], summary="Create a white-label agency profile")
-async def create_agency_endpoint(
- name: str = Body(...),
- owner_email: str = Body(...),
- custom_domain: str = Body(""),
- brand_color: str = Body("#f59e0b"),
- logo_url: str = Body(""),
-) -> dict[str, Any]:
- """Create a white-label agency profile for reselling Pry.
-
- Agencies get:
- - Custom branding (colors, logo, domain)
- - Client management with sub-accounts
- - Usage analytics and quota management
- - API key management for each client
- """
- from agency import create_agency
-
- result = create_agency(name, owner_email, custom_domain, brand_color, logo_url)
- return {"success": result["success"], "data": result}
-@app.get("/v1/agency/{agency_id}", tags=["Agency"], summary="Get agency profile")
-async def get_agency_endpoint(agency_id: str) -> dict[str, Any]:
- """Get agency profile details."""
- from agency import get_agency
-
- result = get_agency(agency_id)
- if not result:
- raise NotFoundError(f"Agency not found: {agency_id}")
- return {"success": True, "data": result}
-@app.put("/v1/agency/{agency_id}/branding", tags=["Agency"], summary="Update agency branding")
-async def update_branding(
- agency_id: str,
- name: str | None = Body(None),
- brand_color: str | None = Body(None),
- logo_url: str | None = Body(None),
- custom_domain: str | None = Body(None),
-) -> dict[str, Any]:
- """Update white-label branding for an agency."""
- from agency import update_agency_branding
-
- result = update_agency_branding(agency_id, name, brand_color, logo_url, custom_domain)
- return {"success": result["success"], "data": result}
-@app.post("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account")
-async def create_client_endpoint(
- agency_id: str,
- client_name: str = Body(...),
- client_email: str = Body(...),
- monthly_quota: int = Body(10000),
-) -> dict[str, Any]:
- """Create a client sub-account under an agency.
-
- Each client gets their own API key and usage quota.
- """
- from agency import create_client
-
- result = create_client(agency_id, client_name, client_email, monthly_quota)
- return {"success": result["success"], "data": result}
-@app.get("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="List agency clients")
-async def list_clients_endpoint(agency_id: str) -> dict[str, Any]:
- """List all clients under an agency."""
- from agency import list_clients
-
- clients = list_clients(agency_id)
- return {"success": True, "data": {"clients": clients, "total": len(clients)}}
-@app.get("/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics")
-async def get_analytics(agency_id: str) -> dict[str, Any]:
- """Get aggregate usage analytics for an agency."""
- from agency import get_agency_analytics
-
- result = get_agency_analytics(agency_id)
- return {"success": True, "data": result}
-@app.get("/v1/client/{client_id}/quota", tags=["Agency"], summary="Check client quota usage")
-async def check_quota(client_id: str) -> dict[str, Any]:
- """Check a client's current quota usage and remaining capacity."""
- from agency import check_client_quota
-
- result = check_client_quota(client_id)
- return {"success": result["success"], "data": result}
# โโ SEO Content Change Monitor โโ
-@app.post("/v1/seo/analyze", tags=["SEO"], summary="Analyze SEO elements from a URL")
-async def seo_analyze(url: str = Body(...)) -> dict[str, Any]:
- """Analyze all SEO elements from a URL.
-
- Returns: title, meta description, keywords, headings (H1/H2),
- canonical, OG tags, Twitter cards, word count, link counts,
- schema markup, hreflang tags.
- """
- from seo_monitor import analyze_seo
-
- result = await analyze_seo(url)
- return {"success": "error" not in result, "data": result}
-@app.post("/v1/seo/track", tags=["SEO"], summary="Track SEO changes since last scan")
-async def seo_track(url: str = Body(...)) -> dict[str, Any]:
- """Track SEO changes since the last scan of this URL.
-
- Compares current SEO elements to previous snapshot and reports
- what changed (title, description, headings, etc.).
- """
- from seo_monitor import track_seo_changes
-
- result = await track_seo_changes(url)
- return {"success": "error" not in result, "data": result}
-@app.post("/v1/seo/keywords", tags=["SEO"], summary="Analyze keyword presence in URL content")
-async def seo_keywords(
- url: str = Body(...),
- keywords: list[str] = Body(...),
-) -> dict[str, Any]:
- """Analyze which keywords a URL targets.
-
- Checks each keyword for:
- - Presence in title tag
- - Presence in H1 headings
- - Presence in meta description
- - Frequency in body content
- - Keyword density percentage
- """
- from seo_monitor import get_seo_keyword_insights
-
- result = await get_seo_keyword_insights(url, keywords)
- return {"success": "error" not in result, "data": result}
# โโ Reports โโ
-@app.post(
- "/v1/reports/generate",
- tags=["Reports"],
- summary="Generate a white-label report from scraped data",
-)
-async def generate_report_endpoint(
- report_type: str = Body(...),
- data: dict[str, Any] = Body(...),
- branding: dict[str, Any] | None = Body(None),
- output_format: str = Body("html"),
-) -> dict[str, Any]:
- """Generate a white-label report from scraped data.
-
- Report types:
- - competitive_analysis: Competitor pricing and activity overview
- - price_monitor: Product price change tracking with visual indicators
- - seo_audit: SEO element analysis with change detection
- - content_tracker: Content change monitoring across pages
-
- Branding (optional): {"agency_name": "...", "brand_color": "#hex", "logo_url": "..."}
- """
- from reports import generate_report
-
- result = generate_report(report_type, data, branding, output_format)
- if "error" in result:
- raise InvalidRequestError(result["error"])
- return {"success": True, "data": result}
-@app.get("/v1/reports", tags=["Reports"], summary="List generated reports")
-async def list_reports_endpoint() -> dict[str, Any]:
- """List all previously generated reports."""
- from reports import list_reports
-
- reports = list_reports()
- return {"success": True, "data": {"reports": reports, "total": len(reports)}}
-@app.get("/v1/report/{report_id}", tags=["Reports"], summary="Get a generated report")
-async def get_report(report_id: str) -> Any:
- """Get the HTML content of a generated report."""
- from reports import REPORTS_DIR
-
- for path in REPORTS_DIR.glob(f"{report_id}_*.html"):
- return HTMLResponse(content=path.read_text())
- raise NotFoundError(f"Report not found: {report_id}")
# โโ Enrichment โโ
-@app.post(
- "/v1/enrich",
- tags=["Enrichment"],
- summary="Enrich scraped data with company info, tech stack, social profiles",
-)
-async def enrich_data(
- url: str = Body(...),
- include_tech_stack: bool = Body(True),
- include_social: bool = Body(True),
- include_company: bool = Body(True),
-) -> dict[str, Any]:
- """Enrich a URL with supplemental business intelligence.
-
- Returns:
- - Tech stack detection (CMS, framework, CDN, analytics, payments)
- - Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.)
- - Company information (email, phone, address, founded year, team size)
- """
- from enrichment import enrich_url
-
- result = await enrich_url(url)
- filtered: dict[str, Any] = {"url": url}
- if include_tech_stack:
- filtered["tech_stack"] = result.get("tech_stack")
- if include_social:
- filtered["social_profiles"] = result.get("social_profiles")
- if include_company:
- filtered["company_info"] = result.get("company_info")
-
- return {"success": True, "data": filtered}
-@app.post(
- "/v1/enrich/tech-stack", tags=["Enrichment"], summary="Detect technologies used on a website"
-)
-async def detect_tech(url: str = Body(...)) -> dict[str, Any]:
- """Detect what technologies a website is built with.
-
- Detects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails),
- frontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar),
- payments (Stripe, PayPal).
- """
- from enrichment import enrich_url
-
- result = await enrich_url(url)
- return {"success": True, "data": result.get("tech_stack", {})}
# โโ AI Agent Integration โโ
-@app.get(
- "/openapi.json",
- tags=["System"],
- summary="OpenAPI spec for AI agent integration",
- include_in_schema=False,
-)
-async def get_openapi() -> JSONResponse:
- """Get the OpenAPI specification for AI agent integration.
-
- Use this with ChatGPT GPT Actions, Claude MCP, or any AI agent
- framework to let AI models scrape the web through Pry.
- """
- from ai_plugin import get_openapi_spec
-
- spec = get_openapi_spec()
- return JSONResponse(content=spec)
-@app.get("/v1/ai/gpt-manifest", tags=["AI"], summary="Get GPT Action manifest for ChatGPT")
-async def get_gpt_manifest() -> JSONResponse:
- """Get the GPT Action manifest for ChatGPT integration.
-
- Add this to your GPT configuration in ChatGPT to give it
- web scraping capabilities through Pry.
- """
- from ai_plugin import get_gpt_action_manifest
-
- return JSONResponse(content=get_gpt_action_manifest())
-@app.get("/v1/ai/mcp-config", tags=["AI"], summary="Get MCP server config for Claude/Cursor")
-async def get_mcp_config() -> JSONResponse:
- """Get the MCP server configuration for Claude/Cursor.
-
- Add this to your AI tool's MCP configuration file to let
- Claude and Cursor scrape the web through Pry.
- """
- from ai_plugin import get_mcp_server_config
-
- return JSONResponse(content={"success": True, "data": get_mcp_server_config()})
-@app.get(
- "/v1/referrals/catalog",
- tags=["Referrals"],
- summary="Get all available referral/affiliate programs",
-)
-async def get_referral_catalog(category: str = "") -> dict[str, Any]:
- """List all referral programs Pry supports.
-
- 60+ providers across categories: LLM, hosting, domains, CDN, email,
- monitoring, proxies, voice, media, devtools, search, CAPTCHA.
- """
- from referrals import ReferralTracker
-
- return {"success": True, "data": ReferralTracker().get_catalog(category)}
-@app.get(
- "/v1/referrals/stats", tags=["Referrals"], summary="Get referral click and conversion stats"
-)
-async def get_referral_stats(days_back: int = 30) -> dict[str, Any]:
- """Get referral tracking statistics for the last N days."""
- from referrals import ReferralTracker
-
- return {"success": True, "data": ReferralTracker().get_stats(days_back)}
-@app.post("/v1/referrals/click", tags=["Referrals"], summary="Record a referral link click")
-async def record_referral_click(
- provider_tag: str = Body(...),
- source: str = Body("api"),
- user_id: str = Body(""),
-) -> dict[str, Any]:
- """Record when a user clicks a referral link. Returns tracking ID."""
- from referrals import PROVIDER_CATALOG, ReferralTracker
-
- url = ""
- for _cat, providers in PROVIDER_CATALOG.items():
- for p in providers:
- if p.get("tag") == provider_tag:
- url = p["url"]
- break
- if url:
- break
- if not url:
- return {"success": False, "error": f"Unknown provider: {provider_tag}"}
- click_id = ReferralTracker().record_click(provider_tag, url, source, user_id)
- return {"success": True, "data": {"click_id": click_id, "url": url}}
-@app.post("/v1/referrals/convert", tags=["Referrals"], summary="Record a referral conversion")
-async def record_referral_conversion(
- click_id: str = Body(...),
- revenue_usd: float = Body(0.0),
- notes: str = Body(""),
-) -> dict[str, Any]:
- """Record that a referral click resulted in a conversion."""
- from referrals import ReferralTracker
-
- success = ReferralTracker().record_conversion(click_id, revenue_usd, notes)
- return {"success": success}
-@app.get("/v1/x402/pricing", tags=["x402"], summary="Get x402 pricing for all paid operations")
-async def x402_pricing() -> dict[str, Any]:
- """Get the price list for pay-per-scrape operations."""
- from x402 import X402Handler
-
- return {"success": True, "data": X402Handler().get_stats()}
-@app.post("/v1/x402/payment", tags=["x402"], summary="Create a x402 payment request")
-async def x402_payment_request(
- operation: str = Body(...),
- metadata: dict[str, Any] | None = Body(None),
-) -> dict[str, Any]:
- """Create a x402 payment request for a paid operation.
-
- Returns payment details (wallet, amount, asset) for the client to pay.
- """
- from x402 import create_payment_request
-
- req = create_payment_request(operation, metadata)
- return {"success": True, "data": req}
-@app.post("/v1/x402/verify", tags=["x402"], summary="Verify a x402 payment was settled")
-async def x402_verify_payment(
- payment_id: str = Body(...),
- tx_hash: str = Body(...),
- network: str = Body(""),
- asset: str = Body(""),
- amount_usd: float = Body(0.0),
-) -> dict[str, Any]:
- """Verify a x402 payment has been settled on-chain via facilitator router."""
- from x402 import X402Handler
-
- result = await X402Handler().verify_payment(
- payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd
- )
- return {"success": result["verified"], "data": result}
-@app.post(
- "/v1/x402/require-payment", tags=["x402"], summary="Generate a 402 Payment Required response"
-)
-async def x402_require_payment(
- operation: str = Body(...),
- metadata: dict[str, Any] | None = Body(None),
-) -> dict[str, Any]:
- """Generate a 402 Payment Required response for a paid endpoint."""
- from x402 import require_payment_legacy
-
- return require_payment_legacy(operation, metadata)
-@app.post("/v1/x402/batch-payment", tags=["x402"], summary="Create a batch x402 payment")
-async def x402_batch_payment(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
- """Create a single x402 payment covering multiple operations.
-
- Returns a PaymentRequired body with the combined amount. After paying,
- submit the tx to POST /v1/x402/batch-verify.
- """
- from x402 import create_batch_payment
-
- operations = payload.get("operations", [])
- if not isinstance(operations, list):
- raise InvalidRequestError("operations must be a list")
- result = await create_batch_payment(operations)
- if "error" in result:
- return {"success": False, "error": result["error"]}
- return {"success": True, "data": result}
-@app.post("/v1/x402/batch-verify", tags=["x402"], summary="Verify a batch x402 payment")
-async def x402_batch_verify(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
- """Verify the on-chain payment for a batch and mark it paid."""
- from x402 import verify_batch_payment
-
- batch_id = payload.get("batch_id", "")
- tx_hash = payload.get("tx_hash", "")
- network = payload.get("network", "")
- asset = payload.get("asset", "")
- if not batch_id or not tx_hash:
- raise InvalidRequestError("batch_id and tx_hash are required")
- result = await verify_batch_payment(batch_id, tx_hash, network=network, asset=asset)
- return {"success": result["verified"], "data": result}
# โโ Proxy Provider & Affiliate Signup Flow โโ
-@app.get("/v1/proxy/providers", tags=["Proxy"], summary="List all available proxy providers")
-async def list_proxy_providers() -> dict[str, Any]:
- """List free + premium proxy providers with affiliate details."""
- from proxy_manager import ProxyManager
-
- return {"success": True, "data": ProxyManager().list_providers()}
-@app.post("/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider")
-async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]:
- """Get the affiliate signup URL for a proxy provider.
-
- Records a click for revenue tracking. The user can then sign up at that URL
- and come back to configure credentials.
- """
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- url = pm.get_signup_link(provider)
- return {"success": True, "data": {"signup_url": url, "provider": provider}}
-@app.get("/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals")
-async def list_proxy_referrals() -> dict[str, Any]:
- """Return the curated proxy provider affiliate catalog (proxy_referrals.py).
-
- This is the marketing-friendly subset: commission, promo codes, tier
- (premium/standard/budget), and referral URLs. Useful for a "Sign up via Pry
- and we get a cut" UI, or for showing users premium options when their free
- proxy fails.
-
- The full connection metadata (host, port, auth) lives in /v1/proxy/providers.
- """
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- return {
- "success": True,
- "data": {
- "providers": pm.list_proxy_referrals(),
- "summary": pm.get_proxy_referral_summary(),
- },
- }
-@app.get(
- "/v1/proxy/referrals/{tag}", tags=["Proxy"], summary="Get a single proxy provider referral"
-)
-async def get_proxy_referral(tag: str) -> dict[str, Any]:
- """Return the affiliate info for a single proxy provider by tag."""
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- info = pm.get_proxy_referral(tag)
- if not info:
- return {"success": False, "error": f"No proxy referral found for tag: {tag}"}
- return {"success": True, "data": {"tag": tag, **info}}
-@app.post("/v1/proxy/configure", tags=["Proxy"], summary="Configure proxy credentials")
-async def proxy_configure(
- provider: str = Body(...),
- username: str = Body(""),
- password: str = Body(""),
- api_key: str = Body(""),
- proxy_url: str = Body(""),
-) -> dict[str, Any]:
- """Configure credentials for a premium proxy provider.
-
- After signing up via /v1/proxy/signup, the user provides their credentials here.
- """
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- creds = {"username": username, "password": password, "api_key": api_key, "proxy_url": proxy_url}
- creds = {k: v for k, v in creds.items() if v}
- result = pm.select_provider(provider, creds)
- return {"success": result["success"], "data": result}
-@app.get("/v1/proxy/test", tags=["Proxy"], summary="Test the active proxy")
-async def proxy_test() -> dict[str, Any]:
- """Test the currently configured proxy and return its public IP."""
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- proxy_url = pm.get_proxy_url()
- if not proxy_url:
- return {"success": True, "data": {"active": False, "message": "No proxy configured"}}
- result = pm.test_proxy(proxy_url)
- return {"success": True, "data": {"active": True, **result}}
-@app.get("/v1/proxy/status", tags=["Proxy"], summary="Get current proxy status")
-async def proxy_status() -> dict[str, Any]:
- """Get current proxy configuration and available providers."""
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- return {
- "success": True,
- "data": {
- "active_config": pm.active_config.__dict__,
- "configured_providers": list(pm.credentials.keys()),
- "available_providers": pm.list_providers(),
- },
- }
-@app.post("/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block")
-async def proxy_recommend(last_error: str = Body("")) -> dict[str, Any]:
- """After a scrape fails with anti-bot detection, get a recommendation
- for which premium proxy provider to sign up with."""
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- rec = pm.get_recommendation(last_error)
- return {"success": True, "data": rec}
-@app.get("/v1/proxy/clicks", tags=["Proxy"], summary="Get recent proxy referral clicks")
-async def proxy_clicks(days_back: int = 30) -> dict[str, Any]:
- """Get recent proxy referral clicks for revenue tracking."""
- from proxy_manager import ProxyManager
-
- pm = ProxyManager()
- clicks = pm.get_recent_clicks(days_back=days_back)
- return {
- "success": True,
- "data": {
- "total_clicks": len(clicks),
- "days_back": days_back,
- "clicks": clicks,
- },
- }
# โโ Advanced Scraping (TLS, GraphQL, Schema.org, WebSocket) โโ
-@app.post(
- "/v1/tls/impersonate",
- tags=["Advanced"],
- summary="Fetch a URL with TLS fingerprint impersonation",
-)
-async def tls_impersonate(
- url: str = Body(...),
- impersonate: str = Body("chrome131"),
- proxy: str = Body(""),
-) -> dict[str, Any]:
- """Fetch a URL while impersonating a real browser's TLS fingerprint.
- Bypasses JA3/JA4 fingerprinting that blocks 80%+ of bot traffic."""
- from tls_fingerprint import TLSScraper
-
- s = TLSScraper()
- if not s.is_available():
- return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"}
- result = await s.fetch(url, impersonate=impersonate, proxy=proxy)
- return result
-@app.post(
- "/v1/tls/rotate",
- tags=["Advanced"],
- summary="Try multiple browser fingerprints until one succeeds",
-)
-async def tls_rotate(
- url: str = Body(...),
- proxy: str = Body(""),
-) -> dict[str, Any]:
- """Try multiple browser fingerprints until one succeeds (anti-fingerprint rotation)."""
- from tls_fingerprint import TLSScraper
-
- s = TLSScraper()
- if not s.is_available():
- return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"}
- result = await s.fetch_with_rotation(url, proxy=proxy)
- return result
-@app.post(
- "/v1/camoufox/fetch", tags=["Advanced"], summary="Fetch with Camoufox anti-detection Firefox"
-)
-async def camoufox_fetch(
- url: str = Body(...),
- profile: str = Body("chrome_windows"),
- wait_selector: str = Body(""),
- proxy: str = Body(""),
-) -> dict[str, Any]:
- """Fetch a URL using Camoufox (Firefox anti-detection browser).
-
- Camoufox patches Firefox at the source level for maximum stealth.
- This is more effective than Playwright for sites with advanced
- fingerprinting (DataDome, PerimeterX, advanced Cloudflare).
- """
- from camoufox_integration import CamoufoxBrowser
-
- b = CamoufoxBrowser()
- if not b.is_available():
- return {
- "success": False,
- "error": "camoufox not installed. Run: pip install camoufox && python -m camoufox fetch",
- }
- result = await b.fetch(url, profile=profile, wait_selector=wait_selector, proxy=proxy)
- return result
-@app.post(
- "/v1/graphql/discover", tags=["Advanced"], summary="Discover GraphQL endpoints for a site"
-)
-async def graphql_discover(base_url: str = Body(...)) -> dict[str, Any]:
- """Auto-discover GraphQL endpoints for a website."""
- from graphql_discovery import GraphQLDiscovery
-
- g = GraphQLDiscovery()
- found = await g.discover(base_url)
- return {"success": True, "data": {"found": found, "count": len(found)}}
-@app.post(
- "/v1/graphql/introspect", tags=["Advanced"], summary="Run GraphQL introspection on an endpoint"
-)
-async def graphql_introspect(endpoint: str = Body(...)) -> dict[str, Any]:
- """Run GraphQL introspection query against a discovered endpoint."""
- from graphql_discovery import GraphQLDiscovery
-
- g = GraphQLDiscovery()
- result = await g.introspect(endpoint)
- return {"success": "error" not in result, "data": result}
-@app.post("/v1/graphql/query", tags=["Advanced"], summary="Execute a GraphQL query")
-async def graphql_query(
- endpoint: str = Body(...),
- query: str = Body(...),
- variables: dict[str, Any] | None = Body(None),
-) -> dict[str, Any]:
- """Execute a GraphQL query against a discovered endpoint."""
- from graphql_discovery import GraphQLDiscovery
-
- g = GraphQLDiscovery()
- result = await g.query(endpoint, query, variables)
- return {"success": "error" not in result, "data": result}
-@app.post(
- "/v1/schema/extract",
- tags=["Advanced"],
- summary="Extract Schema.org structured data from a page",
-)
-async def schema_extract(url: str = Body(...)) -> dict[str, Any]:
- """Extract Schema.org/JSON-LD/Microdata/RDFa structured data from a URL.
- Most modern sites embed structured data โ extract it directly instead of
- scraping HTML. 100x faster and more accurate."""
- from client import get_client
- from schema_extraction import SchemaExtractor
-
- client = await get_client()
- resp = await client.get(url, timeout=30)
- e = SchemaExtractor()
- result = e.extract_all(resp.text)
- return {"success": True, "data": result}
-@app.post(
- "/v1/schema/extract-html",
- tags=["Advanced"],
- summary="Extract Schema.org structured data from raw HTML",
-)
-async def schema_extract_html(html: str = Body(...)) -> dict[str, Any]:
- """Extract Schema.org/JSON-LD/Microdata/RDFa from a raw HTML string.
- Useful when you've already fetched the page and want to extract structured data."""
- from schema_extraction import SchemaExtractor
-
- e = SchemaExtractor()
- result = e.extract_all(html)
- return {"success": True, "data": result}
-@app.post(
- "/v1/ws/scrape",
- tags=["Advanced"],
- summary="Scrape data from a WebSocket endpoint",
-)
-async def ws_scrape(
- url: str = Body(...),
- max_messages: int = Body(100),
- timeout: int = Body(30),
- message_filter: str = Body(""),
-) -> dict[str, Any]:
- """Connect to a WebSocket and capture the data stream.
- Useful for SPAs and real-time apps that load data via WebSocket."""
- from websocket_scraper import WebSocketScraper
-
- s = WebSocketScraper(max_messages=max_messages, timeout=timeout)
- result = await s.scrape_websocket(url, message_filter=message_filter)
- return {"success": result.get("success", False), "data": result}
-@app.post(
- "/v1/sse/scrape",
- tags=["Advanced"],
- summary="Scrape data from a Server-Sent Events endpoint",
-)
-async def sse_scrape(
- url: str = Body(...),
- max_events: int = Body(50),
- timeout: int = Body(30),
- event_filter: str = Body(""),
-) -> dict[str, Any]:
- """Connect to a Server-Sent Events endpoint and capture events."""
- from websocket_scraper import WebSocketScraper
-
- s = WebSocketScraper(timeout=timeout)
- result = await s.scrape_sse(url, event_filter=event_filter, max_events=max_events)
- return {"success": result.get("success", False), "data": result}
# โโ Advanced: Cookie Warming, PDF, OCR, Dedup, Behavior โโ
-@app.post(
- "/v1/cookies/warm",
- tags=["Advanced"],
- summary="Warm cookies for a domain by browsing legitimate pages",
-)
-async def warm_cookies(
- target_domain: str = Body(...),
- pages_to_visit: int = Body(3),
-) -> dict[str, Any]:
- """Pre-age cookies for a domain to bypass anti-bot detection.
- Visits legitimate pages first to build realistic browsing history."""
- from cookie_warmer import CookieWarmer
-
- w = CookieWarmer()
- result = await w.warm_for_site(target_domain, pages_to_visit=pages_to_visit)
- return {"success": result.get("success", False), "data": result}
-@app.get(
- "/v1/cookies/sessions",
- tags=["Advanced"],
- summary="List all warmed cookie sessions",
-)
-async def list_cookie_sessions() -> dict[str, Any]:
- from cookie_warmer import CookieWarmer
-
- w = CookieWarmer()
- return {"success": True, "data": w.list_sessions()}
-@app.post(
- "/v1/pdf/extract",
- tags=["Advanced"],
- summary="Extract tables and text from a PDF",
-)
-async def extract_pdf(
- pdf_url: str = Body(...),
- method: str = Body("pdfplumber"),
-) -> dict[str, Any]:
- """Download a PDF and extract structured tables and text."""
- from client import get_client
- from pdf_extractor import PDFTableExtractor
-
- client = await get_client()
- try:
- resp = await client.get(pdf_url, timeout=60)
- if not resp.is_success:
- return {
- "success": False,
- "error": f"Failed to download: HTTP {resp.status_code}",
- }
- e = PDFTableExtractor()
- result = e.extract(resp.content, method=method)
- return {"success": "error" not in result, "data": result}
- except (httpx.HTTPError, httpx.RequestError) as e:
- return {"success": False, "error": str(e)[:300]}
-@app.post(
- "/v1/ocr/extract",
- tags=["Advanced"],
- summary="Extract text from an image using Tesseract",
-)
-async def ocr_extract(
- image_url: str = Body(""),
- image_base64: str = Body(""),
-) -> dict[str, Any]:
- """Extract text from an image via URL or base64."""
- from ocr_extractor import ImageOCR
-
- o = ImageOCR()
- if image_url:
- result = await o.extract_from_url(image_url)
- elif image_base64:
- data = base64.b64decode(image_base64.split(",", 1)[-1])
- result = o.extract_from_bytes(data)
- else:
- return {"success": False, "error": "Provide image_url or image_base64"}
- return result
-@app.post(
- "/v1/dedup/check",
- tags=["Advanced"],
- summary="Check if content is a near-duplicate using SimHash",
-)
-async def check_duplicate(
- text: str = Body(...),
- threshold: float = Body(0.85),
-) -> dict[str, Any]:
- """Check if text is a near-duplicate of recently seen content using SimHash."""
- from dedup import SimHash
-
- h = SimHash.hash(text)
- return {
- "success": True,
- "data": {
- "hash": h,
- "text_length": len(text),
- "threshold": threshold,
- },
- }
-@app.get(
- "/v1/behavior/simulate",
- tags=["Advanced"],
- summary="Generate human-like behavior patterns for testing",
-)
-async def get_behavior_simulation(action: str = "mouse") -> dict[str, Any]:
- """Generate realistic human behavior patterns (mouse path, scroll, typing, etc.)"""
- from behavioral_biometrics import behavior
-
- if action == "mouse":
- path = behavior.mouse_path((0, 0), (800, 600))
- elif action == "scroll":
- path = behavior.scroll_pattern(3000)
- elif action == "typing":
- path = behavior.typing_pattern("the quick brown fox jumps over")
- elif action == "click":
- return {
- "success": True,
- "data": {"delay_ms": behavior.click_decision_delay() * 1000},
- }
- else:
- return {
- "success": False,
- "error": f"Unknown action: {action}. Use mouse|scroll|typing|click",
- }
- return {"success": True, "data": path}
# โโ x402 payment processing โโ
-@app.post(
- "/v1/x402/pay",
- tags=["x402"],
- summary="Process x402 payment and get access token",
-)
-async def x402_pay(
- operation: str = Body(...),
- tx_hash: str = Body(...),
- payer_wallet: str = Body(...),
- network: str = Body(""),
- asset: str = Body(""),
- amount_usd: float = Body(0.0),
-) -> dict[str, Any]:
- """Process an x402 payment and return an access token.
-
- Flow:
- 1. User gets 402 from a paid endpoint
- 2. User sends USDC to the wallet in the 402 response
- 3. User calls this endpoint with the tx_hash
- 4. Pry verifies the transaction through the facilitator router
- 5. Returns access token (payment_id) to use in X-Payment-Hash header
- """
- from x402 import X402Handler
-
- h = X402Handler()
- payment_id = uuid.uuid4().hex[:12]
- verify_result = await h.verify_payment(
- payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd
- )
- settlement = None
- if verify_result.get("verified"):
- settlement = await h.settle_payment(
- payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd
- )
- return {
- "success": verify_result.get("verified", False),
- "data": {
- "payment_id": payment_id,
- "tx_hash": tx_hash,
- "verified": verify_result,
- "settlement": settlement,
- "payer_wallet": payer_wallet,
- "use_in_header": {"X-Payment-ID": payment_id, "X-Payment-Hash": tx_hash},
- },
- }
# โโ Actor marketplace โโ
-@app.post(
- "/v1/actors/create",
- tags=["Marketplace"],
- summary="Create a new actor",
-)
-async def create_actor(
- name: str = Body(...),
- description: str = Body(...),
- template_id: str = Body(""),
- code: str = Body(""),
- price_per_run: float = Body(0.0),
- visibility: str = Body("private"),
- schedule_cron: str = Body(""),
- tags: list[str] | None = Body(None),
-) -> dict[str, Any]:
- """Create a new actor in the marketplace."""
- from actor_marketplace import ActorMarketplace, ActorVisibility
-
- m = ActorMarketplace()
- actor = m.create(
- name,
- description,
- template_id,
- code,
- price_per_run,
- ActorVisibility(visibility),
- schedule_cron,
- tags or [],
- )
- return {"success": True, "data": actor.to_dict()}
-@app.get(
- "/v1/actors",
- tags=["Marketplace"],
- summary="List actors in the marketplace",
-)
-async def list_actors(visibility: str = "", tag: str = "") -> dict[str, Any]:
- """List actors. Filter by visibility (public/private) and tag."""
- from actor_marketplace import ActorMarketplace
-
- m = ActorMarketplace()
- return {"success": True, "data": m.list(visibility, tag)}
-@app.post(
- "/v1/actors/{actor_id}/run",
- tags=["Marketplace"],
- summary="Run an actor",
-)
-async def run_actor(actor_id: str, inputs: dict[str, Any] | None = Body(None)) -> dict[str, Any]:
- """Run an actor with the provided inputs."""
- from actor_marketplace import ActorMarketplace
-
- m = ActorMarketplace()
- return await m.run(actor_id, inputs or {})
# โโ Webhook delivery โโ
-@app.post(
- "/v1/webhooks/test",
- tags=["Webhooks"],
- summary="Test webhook delivery",
-)
-async def test_webhook(
- url: str = Body(...),
- payload: dict[str, Any] | None = Body(None),
-) -> dict[str, Any]:
- """Test webhook delivery to a URL with HMAC signing."""
- from webhook_delivery import WebhookDelivery
-
- w = WebhookDelivery()
- return await w.deliver(url, payload or {"test": True, "timestamp": "now"}, "test.event")
-@app.get(
- "/v1/webhooks/dead-letter",
- tags=["Webhooks"],
- summary="Get failed webhook deliveries",
-)
-async def get_dead_letter() -> dict[str, Any]:
- """Get the dead letter queue of failed webhook deliveries."""
- from webhook_delivery import WebhookDelivery
-
- w = WebhookDelivery()
- return {"success": True, "data": w.retry_dead_letter()}
if __name__ == "__main__":
diff --git a/cli.py b/cli.py
index 2ec522c..9f9d976 100755
--- a/cli.py
+++ b/cli.py
@@ -83,6 +83,7 @@ def _spinner(label: str):
if stop:
break
print(f" {CYAN}{c}{NC} {label}... ", end="\r", flush=True)
+ # CLI spinner runs in a daemon thread; sync sleep is acceptable here.
time.sleep(0.1)
t = threading.Thread(target=_spin, daemon=True)
diff --git a/docker-compose.yml b/docker-compose.yml
index 9d5765c..d083265 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -80,3 +80,40 @@ services:
volumes:
pry-data:
pry-sessions:
+
+ redis:
+ image: redis:7-alpine
+ container_name: pry-redis
+ restart: unless-stopped
+ ports:
+ - "127.0.0.1:6379:6379"
+ volumes:
+ - pry-redis:/data
+ deploy:
+ resources:
+ limits:
+ memory: 512M
+ cpus: "0.5"
+
+ postgres:
+ image: postgres:15-alpine
+ container_name: pry-postgres
+ restart: unless-stopped
+ ports:
+ - "127.0.0.1:5432:5432"
+ volumes:
+ - pry-postgres:/var/lib/postgresql/data
+ environment:
+ - POSTGRES_USER=pry
+ - POSTGRES_PASSWORD=pry
+ - POSTGRES_DB=pry
+ deploy:
+ resources:
+ limits:
+ memory: 512M
+ cpus: "0.5"
+
+volumes:
+ pry-sessions:
+ pry-redis:
+ pry-postgres:
diff --git a/llm_providers/registry.py b/llm_providers/registry.py
index f4cee3d..579cf0d 100644
--- a/llm_providers/registry.py
+++ b/llm_providers/registry.py
@@ -94,6 +94,35 @@ class LLMRegistry:
break
raise Exception(f"All LLM providers failed. Last: {last_error}")
+ async def complete_or_empty(
+ self,
+ prompt: str,
+ system: str = "",
+ provider_name: str = "",
+ max_tokens: int = 1024,
+ temperature: float = 0.7,
+ model: str = "",
+ fallback: bool = True,
+ ) -> LLMResponse:
+ """Complete via LLM, returning an empty response on total failure.
+
+ Use this at API boundaries where a 500 is worse than degraded output.
+ """
+ try:
+ return await self.complete(
+ prompt, system, provider_name, max_tokens, temperature, model, fallback
+ )
+ except Exception as e: # noqa: BLE001
+ logger.warning("llm_complete_failed_all_providers", extra={"error": str(e)[:120]})
+ return LLMResponse(
+ text="",
+ provider="none",
+ model="",
+ input_tokens=0,
+ output_tokens=0,
+ cost_usd=0.0,
+ )
+
async def embed(self, text: str, provider_name: str = "", model: str = "") -> list[float]:
names = [provider_name] if provider_name else list(self.fallback_chain)
for name in names:
diff --git a/openapi.json b/openapi.json
index 0ec45cc..83291b3 100644
--- a/openapi.json
+++ b/openapi.json
@@ -6,4454 +6,6 @@
"version": "3.0.0"
},
"paths": {
- "/health": {
- "get": {
- "tags": [
- "Health"
- ],
- "summary": "Full health check with dependency status",
- "description": "Comprehensive health check \u2014 probes Ollama, FlareSolverr, Redis.",
- "operationId": "health_check_health_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {}
- }
- }
- }
- }
- }
- },
- "/live": {
- "get": {
- "tags": [
- "Health"
- ],
- "summary": "Kubernetes liveness probe",
- "description": "Simple liveness \u2014 always returns 200 if the process is running.",
- "operationId": "live_live_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": {
- "type": "string"
- },
- "type": "object",
- "title": "Response Live Live Get"
- }
- }
- }
- }
- }
- }
- },
- "/ready": {
- "get": {
- "tags": [
- "Health"
- ],
- "summary": "Kubernetes readiness probe",
- "description": "Readiness \u2014 checks critical dependencies.",
- "operationId": "ready_ready_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {}
- }
- }
- }
- }
- }
- },
- "/v0/stats": {
- "get": {
- "tags": [
- "Stats"
- ],
- "summary": "Get cache, rate limiter, and session stats",
- "operationId": "stats_v0_stats_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Stats V0 Stats Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/costing/dashboard": {
- "get": {
- "tags": [
- "Costing"
- ],
- "summary": "Get cost analytics dashboard",
- "description": "Get the full cost analytics dashboard.\n\nIncludes:\n- Monthly spend breakdown by operation type\n- Projected end-of-month cost\n- Daily spend for last 7 days\n- Cache efficiency metrics\n- Smart schedule recommendations",
- "operationId": "cost_dashboard_v1_costing_dashboard_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Cost Dashboard V1 Costing Dashboard Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/costing/usage": {
- "get": {
- "tags": [
- "Costing"
- ],
- "summary": "Get usage breakdown",
- "description": "Get detailed usage breakdown for a specific month.",
- "operationId": "cost_usage_v1_costing_usage_get",
- "parameters": [
- {
- "name": "year",
- "in": "query",
- "required": false,
- "schema": {
- "anyOf": [
- {
- "type": "integer"
- },
- {
- "type": "null"
- }
- ],
- "title": "Year"
- }
- },
- {
- "name": "month",
- "in": "query",
- "required": false,
- "schema": {
- "anyOf": [
- {
- "type": "integer"
- },
- {
- "type": "null"
- }
- ],
- "title": "Month"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Cost Usage V1 Costing Usage Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/costing/record": {
- "post": {
- "tags": [
- "Costing"
- ],
- "summary": "Record a usage event",
- "description": "Record a usage event for cost tracking.\n\nOperations: scrape_direct, scrape_flaresolverr, scrape_playwright,\ncrawl_page, llm_call, vision_call, extraction_css, bandwidth_mb",
- "operationId": "record_usage_endpoint_v1_costing_record_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_record_usage_endpoint_v1_costing_record_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Record Usage Endpoint V1 Costing Record Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/costing/costs": {
- "post": {
- "tags": [
- "Costing"
- ],
- "summary": "Update per-operation cost table",
- "description": "Update the per-operation cost table with custom prices.",
- "operationId": "update_costs_v1_costing_costs_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": {
- "type": "number"
- },
- "type": "object",
- "title": "Costs"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Update Costs V1 Costing Costs Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/freshness/check": {
- "post": {
- "tags": [
- "Freshness"
- ],
- "summary": "Check if content has changed since last scrape",
- "description": "Check if content has changed since the last scrape.\n\nUses content fingerprinting (SHA256) to detect changes.\nIf no content is provided, does a quick HEAD check instead.",
- "operationId": "freshness_check_v1_freshness_check_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_freshness_check_v1_freshness_check_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Freshness Check V1 Freshness Check Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/freshness/frequency": {
- "post": {
- "tags": [
- "Freshness"
- ],
- "summary": "Get adaptive scrape frequency recommendation",
- "description": "Get an adaptive scrape frequency recommendation based on content volatility.\n\nVolatile pages (frequent changes) get shorter intervals.\nStable pages get longer intervals to save costs.",
- "operationId": "freshness_frequency_v1_freshness_frequency_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_freshness_frequency_v1_freshness_frequency_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Freshness Frequency V1 Freshness Frequency Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/freshness/dashboard": {
- "get": {
- "tags": [
- "Freshness"
- ],
- "summary": "Get content staleness dashboard",
- "description": "Get the content staleness dashboard.\n\nShows all tracked URLs, their last check time, age, and whether\nthey're stale (not checked in 24h).",
- "operationId": "freshness_dashboard_v1_freshness_dashboard_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Freshness Dashboard V1 Freshness Dashboard Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/scrape": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Scrape a single URL",
- "description": "Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.",
- "operationId": "scrape_v1_scrape_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/ScrapeRequest"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Scrape V1 Scrape Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/detect-block": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Detect if a site is blocking the scraper",
- "description": "Detect what kind of anti-bot protection a site is using.\n\nReturns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.\nUseful for debugging scraping issues.",
- "operationId": "detect_block_v1_detect_block_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Detect Block V1 Detect Block Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/ultimate-scrape": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Scrape with 10-tier anti-bot fallback system",
- "description": "Scrape any URL using Pry's ultimate 10-tier anti-detection system.\n\nAutomatically tries: direct \u2192 cloudscraper \u2192 FlareSolverr \u2192\nundetected-chromedriver \u2192 Playwright \u2192 Googlebot \u2192 Archive.org \u2192 Google Cache\n\nReturns the first successful result with the method used.",
- "operationId": "ultimate_scrape_v1_ultimate_scrape_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_ultimate_scrape_v1_ultimate_scrape_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Ultimate Scrape V1 Ultimate Scrape Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/capture/network": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Extract API calls and network patterns from a page",
- "description": "Extract API calls, GraphQL queries, and network patterns from a page.\n\nUseful for understanding how SPAs load data and finding hidden API endpoints.",
- "operationId": "capture_network_v1_capture_network_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Capture Network V1 Capture Network Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/capture/lazy": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Detect and handle lazy-loaded content",
- "description": "Detect lazy loading and infinite scroll patterns on a page.\n\nOptionally generate JS to auto-scroll and load all content.",
- "operationId": "detect_lazy_content_v1_capture_lazy_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_detect_lazy_content_v1_capture_lazy_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Detect Lazy Content V1 Capture Lazy Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/crawl": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Crawl multiple pages from a URL",
- "description": "Crawl multiple pages from a URL. Supports async webhooks.",
- "operationId": "crawl_v1_crawl_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/CrawlRequest"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Crawl V1 Crawl Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/crawl/adaptive": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Crawl with adaptive stopping based on content relevance",
- "description": "Crawl a website with adaptive stopping.\n\nUses information foraging theory to decide when to stop:\n- Stops when content relevance drops below threshold\n- Stops when information gain diminishes\n- Respects max_pages and max_depth limits\n- Ideal for targeted data collection (pricing, docs, products)",
- "operationId": "adaptive_crawl_v1_crawl_adaptive_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_adaptive_crawl_v1_crawl_adaptive_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Adaptive Crawl V1 Crawl Adaptive Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/map": {
- "post": {
- "tags": [
- "Scraping"
- ],
- "summary": "Discover URLs on a site",
- "description": "Discover URLs on a site.",
- "operationId": "map_pages_v1_map_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/MapRequest"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Map Pages V1 Map Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/parse": {
- "post": {
- "tags": [
- "Parsing"
- ],
- "summary": "Parse a document (PDF, DOCX, image, CSV, JSON)",
- "description": "Parse a document (PDF, DOCX, image, CSV, JSON) to text.",
- "operationId": "parse_document_v1_parse_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/ParseRequest"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Parse Document V1 Parse Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/markdown": {
- "post": {
- "tags": [
- "Parsing"
- ],
- "summary": "Generate markdown with content filtering strategies",
- "description": "Generate markdown with configurable content filtering.\n\nModes:\n- raw: Unfiltered markdown\n- fit: Prune boilerplate (nav, ads, footers)\n- bm25: Filter by BM25 relevance to query (requires query param)",
- "operationId": "generate_markdown_v1_markdown_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_generate_markdown_v1_markdown_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Generate Markdown V1 Markdown Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/automate": {
- "post": {
- "tags": [
- "Automation"
- ],
- "summary": "Execute browser automation steps",
- "description": "Execute browser automation steps. Sessions persist for login flows.",
- "operationId": "automate_v1_automate_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/AutomateRequest"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Automate V1 Automate Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/screenshot": {
- "post": {
- "tags": [
- "Automation"
- ],
- "summary": "Take a screenshot of a URL",
- "description": "Take a screenshot of a URL. Returns base64 PNG.",
- "operationId": "screenshot_v1_screenshot_post",
- "parameters": [
- {
- "name": "session_id",
- "in": "query",
- "required": false,
- "schema": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Session Id"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_screenshot_v1_screenshot_post"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Screenshot V1 Screenshot Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/vision": {
- "post": {
- "tags": [
- "Vision"
- ],
- "summary": "Analyze an image with a free vision model",
- "description": "Analyze an image with a free vision model.\n\nProvide ONE of: image (base64 or data URI), url (auto-screenshot),\nor file_path (local PNG/JPG).\n\nAuto-falls-back across 5 free OpenRouter vision models if the\nrequested one is rate-limited.",
- "operationId": "vision_v1_vision_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_vision_v1_vision_post"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Vision V1 Vision Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/vision/models": {
- "get": {
- "tags": [
- "Vision"
- ],
- "summary": "List free vision-capable models on OpenRouter",
- "description": "List free vision-capable models on OpenRouter.",
- "operationId": "vision_models_v1_vision_models_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Vision Models V1 Vision Models Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/session/create": {
- "post": {
- "tags": [
- "Sessions"
- ],
- "summary": "Create a persistent browser session",
- "description": "Create a persistent browser session.",
- "operationId": "create_session_v1_session_create_post",
- "parameters": [
- {
- "name": "persist",
- "in": "query",
- "required": false,
- "schema": {
- "type": "boolean",
- "default": true,
- "title": "Persist"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Create Session V1 Session Create Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/session/destroy": {
- "post": {
- "tags": [
- "Sessions"
- ],
- "summary": "Destroy a browser session with optional save",
- "description": "Destroy a browser session. Optionally save state first.",
- "operationId": "destroy_session_v1_session_destroy_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_destroy_session_v1_session_destroy_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Destroy Session V1 Session Destroy Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/sessions": {
- "get": {
- "tags": [
- "Sessions"
- ],
- "summary": "List all persistent sessions",
- "description": "List all persistent sessions (active + saved).",
- "operationId": "list_sessions_v1_sessions_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Sessions V1 Sessions Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/session/save": {
- "post": {
- "tags": [
- "Sessions"
- ],
- "summary": "Save current session state to disk",
- "description": "Save a session's current state (cookies, storage) to disk.",
- "operationId": "save_session_state_v1_session_save_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Session Id"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Save Session State V1 Session Save Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/session/restore": {
- "post": {
- "tags": [
- "Sessions"
- ],
- "summary": "Restore a saved session",
- "description": "Restore a session from disk into a browser context.",
- "operationId": "restore_session_v1_session_restore_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Session Id"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Restore Session V1 Session Restore Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/batch": {
- "post": {
- "tags": [
- "Batch"
- ],
- "summary": "Scrape multiple URLs in parallel",
- "description": "Scrape multiple URLs in parallel. Firecrawl charges extra for batch.",
- "operationId": "batch_scrape_v1_batch_post",
- "parameters": [
- {
- "name": "timeout",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 30,
- "title": "Timeout"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "array",
- "items": {
- "type": "string"
- },
- "title": "Urls"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Batch Scrape V1 Batch Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/compare": {
- "post": {
- "tags": [
- "Analysis"
- ],
- "summary": "Compare content of two URLs",
- "description": "Scrape two URLs and compare their content. Shows additions, deletions, changes.",
- "operationId": "compare_v1_compare_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_compare_v1_compare_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Compare V1 Compare Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/watch": {
- "post": {
- "tags": [
- "Analysis"
- ],
- "summary": "Watch a page for changes",
- "description": "Watch a page for changes. Accepts a webhook URL for notification.\nFirst call registers the watch, subsequent calls compare and notify.\nFirecrawl charges $99/mo for this feature.",
- "operationId": "watch_page_v1_watch_post",
- "parameters": [
- {
- "name": "interval",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 3600,
- "title": "Interval"
- }
- },
- {
- "name": "selector",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "",
- "title": "Selector"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_watch_page_v1_watch_post"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Watch Page V1 Watch Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/export": {
- "post": {
- "tags": [
- "Export"
- ],
- "summary": "Export scraped content in multiple formats",
- "description": "Export scraped content in multiple formats: json, csv, txt, rss.\nFirecrawl only does markdown.",
- "operationId": "export_data_v1_export_post",
- "parameters": [
- {
- "name": "format",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "json",
- "title": "Format"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Export Data V1 Export Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/shadow-dom": {
- "post": {
- "tags": [
- "Parsing"
- ],
- "summary": "Extract content from Shadow DOM",
- "description": "Scrape a page and extract content from Shadow DOM components.\n\nUseful for modern web apps built with Lit, web components, or\nframeworks that use Shadow DOM encapsulation.",
- "operationId": "extract_shadow_dom_v1_shadow_dom_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_extract_shadow_dom_v1_shadow_dom_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Extract Shadow Dom V1 Shadow Dom Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/extract-table": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Extract HTML tables as structured data",
- "description": "Extract HTML tables from a page as structured data.\nFirecrawl doesn't support table extraction at all.",
- "operationId": "extract_table_v1_extract_table_post",
- "parameters": [
- {
- "name": "table_index",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 0,
- "title": "Table Index"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Extract Table V1 Extract Table Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/links": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Analyze links on a page",
- "description": "Analyze all links on a page \u2014 internal, external, broken, social.\nFirecrawl only has basic map functionality.",
- "operationId": "analyze_links_v1_links_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Analyze Links V1 Links Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/seo": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "SEO analysis of a page",
- "description": "SEO analysis of a page: title, description, headings, images, keywords, readability.\nFirecrawl has zero SEO features.",
- "operationId": "analyze_seo_v1_seo_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Analyze Seo V1 Seo Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/summarize": {
- "post": {
- "tags": [
- "Analysis"
- ],
- "summary": "AI summarize scraped content",
- "description": "Scrape + AI summarize using local Ollama. Free, private.",
- "operationId": "summarize_v1_summarize_post",
- "parameters": [
- {
- "name": "max_words",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 100,
- "title": "Max Words"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Summarize V1 Summarize Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/diff": {
- "post": {
- "tags": [
- "Analysis"
- ],
- "summary": "Track page changes over time",
- "description": "Track page changes over time. Returns diff from last scrape.",
- "operationId": "diff_v1_diff_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_diff_v1_diff_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Diff V1 Diff Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/schema": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Extract Schema.org/JSON-LD structured data",
- "description": "Extract Schema.org/JSON-LD structured data from a page.",
- "operationId": "extract_schema_v1_schema_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Extract Schema V1 Schema Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/emails": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Find email addresses on a page",
- "description": "Find all email addresses on a page.",
- "operationId": "find_emails_v1_emails_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Find Emails V1 Emails Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/email/scrape": {
- "post": {
- "tags": [
- "Email"
- ],
- "summary": "Extract data from an email (subject + body)",
- "description": "Extract structured data from an email subject and body.\n\nAuto-classifies as: order_confirmation, invoice, receipt,\nshipping_notification, subscription, or other.\n\nExtracts: order numbers, amounts, tracking numbers, dates, addresses.",
- "operationId": "scrape_email_v1_email_scrape_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_scrape_email_v1_email_scrape_post"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Scrape Email V1 Email Scrape Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/email/gmail": {
- "post": {
- "tags": [
- "Email"
- ],
- "summary": "Fetch and extract data from Gmail inbox",
- "description": "Connect to Gmail and extract structured data from emails.\n\nRequires a Gmail OAuth2 access token with scope:\nhttps://www.googleapis.com/auth/gmail.readonly\n\nExtracts order confirmations, invoices, receipts, shipping notifications.",
- "operationId": "fetch_gmail_v1_email_gmail_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_fetch_gmail_v1_email_gmail_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Fetch Gmail V1 Email Gmail Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/email/outlook": {
- "post": {
- "tags": [
- "Email"
- ],
- "summary": "Fetch and extract data from Outlook inbox",
- "description": "Connect to Outlook/Office 365 and extract structured data from emails.\n\nRequires a Microsoft Graph OAuth2 access token with scope:\nhttps://graph.microsoft.com/Mail.Read\n\nExtracts order confirmations, invoices, receipts, shipping notifications.",
- "operationId": "fetch_outlook_v1_email_outlook_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_fetch_outlook_v1_email_outlook_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Fetch Outlook V1 Email Outlook Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/categorize": {
- "post": {
- "tags": [
- "Analysis"
- ],
- "summary": "AI-categorize scraped content",
- "description": "AI-categorize scraped content into topic tags.",
- "operationId": "categorize_v1_categorize_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Categorize V1 Categorize Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/destinations": {
- "get": {
- "tags": [
- "Integrations"
- ],
- "summary": "List supported data destinations",
- "description": "List all supported data destinations and their config requirements.",
- "operationId": "list_destinations_v1_destinations_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Destinations V1 Destinations Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/destination/send": {
- "post": {
- "tags": [
- "Integrations"
- ],
- "summary": "Send scraped data to a business destination",
- "description": "Send extracted data to a business destination in one click.\n\nDestinations:\n- slack: Send to Slack channel via webhook\n- googlesheets: Write to Google Sheets (requires credentials)\n- airtable: Write to Airtable base (requires API key)\n- email: Send via email (requires SMTP config)\n\nConfig varies by destination:\n- slack: {\"webhook_url\": \"https://hooks.slack.com/...\"}\n- googlesheets: {\"spreadsheet_id\": \"...\", \"credentials_json\": \"...\"}\n- airtable: {\"base_id\": \"...\", \"table_name\": \"Table 1\", \"api_key\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"subject\": \"Data Export\"}",
- "operationId": "send_to_destination_v1_destination_send_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_send_to_destination_v1_destination_send_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Send To Destination V1 Destination Send Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/scrape-and-send": {
- "post": {
- "tags": [
- "Integrations"
- ],
- "summary": "Scrape a URL and send to a destination",
- "description": "Scrape a URL and send the results to a business destination in one step.\n\nCombines /v1/scrape + /v1/destination/send into a single call.\nPerfect for non-technical users who just want data in their tools.",
- "operationId": "scrape_and_send_v1_scrape_and_send_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_scrape_and_send_v1_scrape_and_send_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Scrape And Send V1 Scrape And Send Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/alert/send": {
- "post": {
- "tags": [
- "Alerts"
- ],
- "summary": "Send an alert to any channel",
- "description": "Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email.\n\nConfig varies by channel:\n- slack: {\"webhook_url\": \"...\"}\n- discord: {\"webhook_url\": \"...\"}\n- teams: {\"webhook_url\": \"...\"}\n- telegram: {\"bot_token\": \"...\", \"chat_id\": \"...\"}\n- sms: {\"phone\": \"+1234567890\", \"twilio_sid\": \"...\", \"twilio_token\": \"...\", \"twilio_from\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"smtp_host\": \"...\", \"smtp_user\": \"...\"}",
- "operationId": "send_alert_endpoint_v1_alert_send_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_send_alert_endpoint_v1_alert_send_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Send Alert Endpoint V1 Alert Send Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/alert/channels": {
- "get": {
- "tags": [
- "Alerts"
- ],
- "summary": "List supported alert channels",
- "description": "List all supported alert channels and their config requirements.",
- "operationId": "list_channels_v1_alert_channels_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Channels V1 Alert Channels Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/job/{job_id}": {
- "get": {
- "tags": [
- "Jobs"
- ],
- "summary": "Get async job status and result",
- "description": "Get async job status and result.",
- "operationId": "get_job_v1_job__job_id__get",
- "parameters": [
- {
- "name": "job_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Job Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Job V1 Job Job Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/config": {
- "get": {
- "tags": [
- "Config"
- ],
- "summary": "Get current Pry configuration",
- "description": "Get current Pry configuration.",
- "operationId": "get_config_v1_config_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Get Config V1 Config Get"
- }
- }
- }
- }
- }
- },
- "post": {
- "tags": [
- "Config"
- ],
- "summary": "Update Pry configuration at runtime",
- "description": "Update Pry configuration at runtime.",
- "operationId": "update_config_v1_config_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Updates"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Update Config V1 Config Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/config/profile/tor": {
- "post": {
- "tags": [
- "Config"
- ],
- "summary": "Enable Tor routing for all requests",
- "description": "Enable Tor routing for all requests.",
- "operationId": "enable_tor_v1_config_profile_tor_post",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Enable Tor V1 Config Profile Tor Post"
- }
- }
- }
- }
- }
- }
- },
- "/v1/batch-file": {
- "post": {
- "tags": [
- "Batch"
- ],
- "summary": "Batch scrape URLs from a file with templates",
- "operationId": "batch_from_file_v1_batch_file_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/BatchFileRequest"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Batch From File V1 Batch File Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/record/start": {
- "post": {
- "tags": [
- "Recorder"
- ],
- "summary": "Start recording browser actions",
- "operationId": "start_recording_v1_record_start_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Session Id"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Start Recording V1 Record Start Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/record/step": {
- "post": {
- "tags": [
- "Recorder"
- ],
- "summary": "Record a browser action step",
- "operationId": "record_step_v1_record_step_post",
- "parameters": [
- {
- "name": "selector",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "",
- "title": "Selector"
- }
- },
- {
- "name": "value",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "",
- "title": "Value"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_record_step_v1_record_step_post"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Record Step V1 Record Step Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/record/export": {
- "post": {
- "tags": [
- "Recorder"
- ],
- "summary": "Export recording as script",
- "operationId": "export_recording_v1_record_export_post",
- "parameters": [
- {
- "name": "fmt",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "json",
- "title": "Fmt"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Session Id"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Export Recording V1 Record Export Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/record/clear": {
- "post": {
- "tags": [
- "Recorder"
- ],
- "summary": "Clear recorded actions",
- "operationId": "clear_recording_v1_record_clear_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Session Id"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Clear Recording V1 Record Clear Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/transform": {
- "post": {
- "tags": [
- "Transform"
- ],
- "summary": "Transform data to multiple formats",
- "operationId": "transform_data_v1_transform_post",
- "parameters": [
- {
- "name": "format",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "csv",
- "title": "Format"
- }
- }
- ],
- "requestBody": {
- "required": true,
- "content": {
- "application/json": {
- "schema": {
- "type": "array",
- "items": {
- "type": "object",
- "additionalProperties": true
- },
- "title": "Data"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Transform Data V1 Transform Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/run": {
- "post": {
- "tags": [
- "Execute"
- ],
- "summary": "Execute a Pryfile",
- "operationId": "run_pryfile_v1_run_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Path",
- "default": "pry.yml"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Run Pryfile V1 Run Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/jobs": {
- "get": {
- "tags": [
- "Jobs"
- ],
- "summary": "List jobs in a Pryfile",
- "operationId": "list_jobs_v1_jobs_get",
- "parameters": [
- {
- "name": "path",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "pry.yml",
- "title": "Path"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response List Jobs V1 Jobs Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/dashboard": {
- "get": {
- "tags": [
- "Dashboard"
- ],
- "summary": "Pry health and performance dashboard",
- "operationId": "dashboard_dashboard_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {}
- }
- }
- }
- }
- }
- },
- "/v1/suggest": {
- "post": {
- "tags": [
- "Analysis"
- ],
- "summary": "AI-suggest schema fields for a URL",
- "operationId": "suggest_schema_v1_suggest_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Data"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Suggest Schema V1 Suggest Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/breaker/status": {
- "post": {
- "tags": [
- "Circuit Breaker"
- ],
- "summary": "Get circuit breaker status",
- "operationId": "breaker_status_v1_breaker_status_post",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Breaker Status V1 Breaker Status Post"
- }
- }
- }
- }
- }
- }
- },
- "/v1/breaker/reset": {
- "post": {
- "tags": [
- "Circuit Breaker"
- ],
- "summary": "Reset circuit breaker for a domain",
- "operationId": "breaker_reset_v1_breaker_reset_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Domain",
- "default": ""
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Breaker Reset V1 Breaker Reset Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/extract": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Extract structured fields from a URL",
- "operationId": "extract_stable_v1_extract_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Data"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Extract Stable V1 Extract Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/extract/css": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Extract structured data with CSS selectors (no LLM)",
- "description": "Extract structured JSON from a URL using CSS selector schema.\n\nSchema format:\n{\n \"name\": \"products\",\n \"base_selector\": \".product-card\",\n \"fields\": [\n {\"name\": \"title\", \"selector\": \"h3\", \"type\": \"text\"},\n {\"name\": \"price\", \"selector\": \".price\", \"type\": \"text\", \"transform\": \"float\"},\n {\"name\": \"link\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"},\n {\"name\": \"in_stock\", \"selector\": \".stock\", \"type\": \"exists\"},\n ]\n}",
- "operationId": "extract_css_v1_extract_css_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_extract_css_v1_extract_css_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Extract Css V1 Extract Css Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/extract/llm": {
- "post": {
- "tags": [
- "Extraction"
- ],
- "summary": "Extract with LLM + chunking strategies",
- "description": "Extract structured data using LLM with intelligent chunking.\n\nChunks content by strategy (topic/sentence/regex), optionally filters\nby relevance to query, then extracts from each chunk.",
- "operationId": "extract_llm_v1_extract_llm_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_extract_llm_v1_extract_llm_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Extract Llm V1 Extract Llm Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipeline/hook": {
- "post": {
- "tags": [
- "Pipeline"
- ],
- "summary": "Register a hook function",
- "description": "Register a JavaScript hook function at a pipeline hook point.\n\nHook points: before_scrape, after_response, before_parse, after_parse,\n before_extract, after_extract, before_return, on_error",
- "operationId": "register_hook_v1_pipeline_hook_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_register_hook_v1_pipeline_hook_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Register Hook V1 Pipeline Hook Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipeline/hooks": {
- "get": {
- "tags": [
- "Pipeline"
- ],
- "summary": "List all registered hooks",
- "description": "List all registered hooks in the pipeline.",
- "operationId": "list_pipeline_hooks_v1_pipeline_hooks_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Pipeline Hooks V1 Pipeline Hooks Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipeline/run": {
- "post": {
- "tags": [
- "Pipeline"
- ],
- "summary": "Run pipeline hooks for testing",
- "description": "Run hooks at a given pipeline point for testing.",
- "operationId": "run_pipeline_test_v1_pipeline_run_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_run_pipeline_test_v1_pipeline_run_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Run Pipeline Test V1 Pipeline Run Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipe": {
- "post": {
- "tags": [
- "Transform"
- ],
- "summary": "Scrape and transform via data pipeline",
- "operationId": "data_pipeline_v1_pipe_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Data"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Data Pipeline V1 Pipe Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/share": {
- "post": {
- "tags": [
- "Share"
- ],
- "summary": "Share scraped content via link",
- "operationId": "share_scrape_v1_share_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Data"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Share Scrape V1 Share Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/share/{share_id}": {
- "get": {
- "tags": [
- "Share"
- ],
- "summary": "View shared content",
- "operationId": "view_share_share__share_id__get",
- "parameters": [
- {
- "name": "share_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Share Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {}
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/compliance/check": {
- "get": {
- "tags": [
- "Compliance"
- ],
- "summary": "Get compliance check documentation",
- "description": "Get information about the compliance check endpoint.",
- "operationId": "compliance_docs_v1_compliance_check_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Compliance Docs V1 Compliance Check Get"
- }
- }
- }
- }
- }
- },
- "post": {
- "tags": [
- "Compliance"
- ],
- "summary": "Run full compliance check on a URL",
- "description": "Run a full legal compliance check on a target URL.\n\nAnalyzes:\n- robots.txt crawl permissions\n- Terms of Service classification (restrictive/permissive/moderate)\n- Jurisdiction detection (GDPR, CCPA, LGPD)\n- Sensitive data detection (PII, financial, health, contact)\n\nReturns a green/yellow/red risk score with recommendations.",
- "operationId": "compliance_check_v1_compliance_check_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Compliance Check V1 Compliance Check Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/consent": {
- "post": {
- "tags": [
- "GDPR"
- ],
- "summary": "Record user consent for data processing",
- "description": "Record a user's consent for data processing (GDPR Art. 7).\n\nStores: user hash, purpose, timestamp, IP, user agent.\nConsent expires after 365 days.",
- "operationId": "record_consent_endpoint_v1_gdpr_consent_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_record_consent_endpoint_v1_gdpr_consent_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Record Consent Endpoint V1 Gdpr Consent Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/consent/{user_id}": {
- "get": {
- "tags": [
- "GDPR"
- ],
- "summary": "Check user consent status",
- "description": "Check if a user has given valid consent for data processing.",
- "operationId": "check_consent_endpoint_v1_gdpr_consent__user_id__get",
- "parameters": [
- {
- "name": "user_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "User Id"
- }
- },
- {
- "name": "purpose",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "data_collection",
- "title": "Purpose"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Check Consent Endpoint V1 Gdpr Consent User Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/consent/revoke": {
- "post": {
- "tags": [
- "GDPR"
- ],
- "summary": "Revoke user consent",
- "description": "Revoke a user's consent for a processing purpose (GDPR Art. 7(3)).",
- "operationId": "revoke_consent_endpoint_v1_gdpr_consent_revoke_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_revoke_consent_endpoint_v1_gdpr_consent_revoke_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Revoke Consent Endpoint V1 Gdpr Consent Revoke Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/deletion/request": {
- "post": {
- "tags": [
- "GDPR"
- ],
- "summary": "Request data deletion (right to erasure)",
- "description": "Request deletion of all data associated with a user (GDPR Art. 17).\n\nCreates a deletion request that can be executed immediately or\nafter a holding period.",
- "operationId": "request_deletion_endpoint_v1_gdpr_deletion_request_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_request_deletion_endpoint_v1_gdpr_deletion_request_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Request Deletion Endpoint V1 Gdpr Deletion Request Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/deletion/execute": {
- "post": {
- "tags": [
- "GDPR"
- ],
- "summary": "Execute data deletion request",
- "description": "Execute a pending deletion request, removing all user data.",
- "operationId": "execute_deletion_endpoint_v1_gdpr_deletion_execute_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Request Id"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Execute Deletion Endpoint V1 Gdpr Deletion Execute Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/retention": {
- "get": {
- "tags": [
- "GDPR"
- ],
- "summary": "Get data retention policy",
- "description": "Get the current data retention policy.",
- "operationId": "get_retention_policy_endpoint_v1_gdpr_retention_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Get Retention Policy Endpoint V1 Gdpr Retention Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/retention/apply": {
- "post": {
- "tags": [
- "GDPR"
- ],
- "summary": "Apply retention policy to remove expired data",
- "description": "Apply the retention policy, removing all expired data.",
- "operationId": "apply_retention_v1_gdpr_retention_apply_post",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Apply Retention V1 Gdpr Retention Apply Post"
- }
- }
- }
- }
- }
- }
- },
- "/v1/gdpr/audit": {
- "get": {
- "tags": [
- "GDPR"
- ],
- "summary": "Get compliance audit log",
- "description": "Get the compliance audit log for the specified period.",
- "operationId": "get_audit_log_endpoint_v1_gdpr_audit_get",
- "parameters": [
- {
- "name": "days_back",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 7,
- "title": "Days Back"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Audit Log Endpoint V1 Gdpr Audit Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/training/classify-license": {
- "post": {
- "tags": [
- "Training"
- ],
- "summary": "Classify content license for AI training",
- "description": "Classify the license of scraped content for AI training compliance.\n\nReturns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use),\ntier (permissive/copyleft/restrictive/conditional), and confidence.",
- "operationId": "classify_license_endpoint_v1_training_classify_license_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Text"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Classify License Endpoint V1 Training Classify License Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/training/clean": {
- "post": {
- "tags": [
- "Training"
- ],
- "summary": "Strip PII and copyright from content",
- "description": "Strip PII and copyright content for AI training clean room.\n\nRemoves emails, phones, SSNs, credit cards, IPs, and (optionally) names.\nAlso strips copyright notices and near-verbatim copyright content.",
- "operationId": "clean_content_v1_training_clean_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_clean_content_v1_training_clean_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Clean Content V1 Training Clean Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/training/export": {
- "post": {
- "tags": [
- "Training"
- ],
- "summary": "Export a clean AI training dataset",
- "description": "Export scraped content as a clean AI training dataset.\n\nEach record should have: content, url, and optional metadata.\n\nFeatures:\n- Per-record provenance tracking (source URL, timestamp, extraction method)\n- PII stripping (email, phone, SSN, CC, IP)\n- Copyright verbatim text removal\n- License classification\n- Compliance report generation",
- "operationId": "export_dataset_v1_training_export_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_export_dataset_v1_training_export_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Export Dataset V1 Training Export Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/training/compliance/{dataset_id}": {
- "get": {
- "tags": [
- "Training"
- ],
- "summary": "Generate compliance report for a dataset",
- "description": "Generate a compliance report for an exported training dataset.\n\nReport includes: data provenance, PII/copyright removal stats,\nlicense classification, and legal recommendations.",
- "operationId": "compliance_report_v1_training_compliance__dataset_id__get",
- "parameters": [
- {
- "name": "dataset_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Dataset Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Compliance Report V1 Training Compliance Dataset Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/monitor": {
- "post": {
- "tags": [
- "Monitoring"
- ],
- "summary": "Create a scheduled content monitor",
- "description": "Create a scheduled monitor that tracks content changes.\n\nArgs:\n name: Human-readable monitor name\n url: Target URL to monitor\n schedule_cron: Cron expression (default: every 6 hours)\n goal: Natural language goal for meaningful-change detection\n webhook: URL to notify on meaningful changes\n use_llm: Use LLM for change judging (slower but smarter)",
- "operationId": "create_monitor_endpoint_v1_monitor_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_create_monitor_endpoint_v1_monitor_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Create Monitor Endpoint V1 Monitor Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/monitor/run": {
- "post": {
- "tags": [
- "Monitoring"
- ],
- "summary": "Run a single monitor check",
- "description": "Execute a monitor check immediately (outside of schedule).",
- "operationId": "run_monitor_endpoint_v1_monitor_run_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Monitor Id"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Run Monitor Endpoint V1 Monitor Run Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/monitors": {
- "get": {
- "tags": [
- "Monitoring"
- ],
- "summary": "List all monitors",
- "description": "List all registered monitors.",
- "operationId": "list_monitors_endpoint_v1_monitors_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Monitors Endpoint V1 Monitors Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/monitor/{monitor_id}": {
- "delete": {
- "tags": [
- "Monitoring"
- ],
- "summary": "Delete a monitor",
- "description": "Delete a monitor and its snapshots.",
- "operationId": "delete_monitor_endpoint_v1_monitor__monitor_id__delete",
- "parameters": [
- {
- "name": "monitor_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Monitor Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Delete Monitor Endpoint V1 Monitor Monitor Id Delete"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/monitor/check": {
- "post": {
- "tags": [
- "Monitoring"
- ],
- "summary": "Run all due monitors",
- "description": "Check all monitors and run any that are due based on their cron schedule.",
- "operationId": "run_due_monitors_v1_monitor_check_post",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Run Due Monitors V1 Monitor Check Post"
- }
- }
- }
- }
- }
- }
- },
- "/v1/structure/check": {
- "post": {
- "tags": [
- "Structure"
- ],
- "summary": "Check if CSS selectors still match on a page",
- "description": "Check if CSS selectors still match on a page.\n\nUse this to verify your scraper templates still work after\na site redesign. Returns per-selector match status.\n\nSelectors format: [{\"name\": \"title\", \"selector\": \"h1\", \"type\": \"css\"}]",
- "operationId": "structure_check_v1_structure_check_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_structure_check_v1_structure_check_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Structure Check V1 Structure Check Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/structure/monitor": {
- "post": {
- "tags": [
- "Structure"
- ],
- "summary": "Monitor page structure changes over time",
- "description": "Monitor page structure changes and detect broken selectors.\n\nCompares current selector match status to previous checks.\nAlerts when selectors stop matching (site redesign detected).\n\nUse this as a pre-scrape health check for your templates.",
- "operationId": "structure_monitor_v1_structure_monitor_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_structure_monitor_v1_structure_monitor_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Structure Monitor V1 Structure Monitor Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/structure/check-template": {
- "post": {
- "tags": [
- "Structure"
- ],
- "summary": "Verify a scraper template still works",
- "description": "Check if a pre-built scraper template still works against a URL.\n\nExtracts the template's selectors and checks each one.\nIf selectors fail, the template needs updating.",
- "operationId": "structure_check_template_v1_structure_check_template_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_structure_check_template_v1_structure_check_template_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Structure Check Template V1 Structure Check Template Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/intel/snapshot": {
- "post": {
- "tags": [
- "Intelligence"
- ],
- "summary": "Record a competitor data snapshot",
- "description": "Record a data snapshot for a competitor.\n\nSnapshots are stored with timestamps and used for trend analysis,\nanomaly detection, and report generation.",
- "operationId": "record_intel_snapshot_v1_intel_snapshot_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_record_intel_snapshot_v1_intel_snapshot_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Record Intel Snapshot V1 Intel Snapshot Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/intel/snapshots/{competitor_id}": {
- "get": {
- "tags": [
- "Intelligence"
- ],
- "summary": "Get competitor snapshots",
- "description": "Get historical snapshots for a competitor.",
- "operationId": "get_intel_snapshots_v1_intel_snapshots__competitor_id__get",
- "parameters": [
- {
- "name": "competitor_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Competitor Id"
- }
- },
- {
- "name": "limit",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 50,
- "title": "Limit"
- }
- },
- {
- "name": "since_hours",
- "in": "query",
- "required": false,
- "schema": {
- "anyOf": [
- {
- "type": "integer"
- },
- {
- "type": "null"
- }
- ],
- "title": "Since Hours"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Intel Snapshots V1 Intel Snapshots Competitor Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/intel/analyze": {
- "post": {
- "tags": [
- "Intelligence"
- ],
- "summary": "Analyze competitor field for anomalies",
- "description": "Analyze a specific field across a competitor's snapshots for anomalies.\n\nReturns statistics, z-score analysis, and anomaly detection.",
- "operationId": "analyze_field_v1_intel_analyze_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_analyze_field_v1_intel_analyze_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Analyze Field V1 Intel Analyze Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/intel/report": {
- "post": {
- "tags": [
- "Intelligence"
- ],
- "summary": "Generate a competitive intelligence report",
- "description": "Generate a competitive intelligence report for tracked competitors.\n\nAnalyzes changes over the specified period and returns a structured report\nwith a natural-language summary.",
- "operationId": "generate_report_v1_intel_report_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_generate_report_v1_intel_report_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Generate Report V1 Intel Report Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/quality/check": {
- "post": {
- "tags": [
- "Quality"
- ],
- "summary": "Run data quality check on extraction results",
- "description": "Run a full data quality check on extraction results.\n\nMetrics:\n- Completeness: what % of expected fields have values\n- Schema adherence: field types match expectations\n- Freshness: how old is the data\n- Anomaly detection: what changed since last extraction\n\nUse this to validate data BEFORE sending to downstream systems.",
- "operationId": "quality_check_v1_quality_check_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_quality_check_v1_quality_check_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Quality Check V1 Quality Check Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/quality/stats": {
- "get": {
- "tags": [
- "Quality"
- ],
- "summary": "Get quality statistics for all checked URLs",
- "description": "Get aggregate quality statistics across all checked URLs.",
- "operationId": "quality_stats_v1_quality_stats_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Quality Stats V1 Quality Stats Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/reconcile": {
- "post": {
- "tags": [
- "Reconciliation"
- ],
- "summary": "Reconcile records from multiple sources into unified entities",
- "description": "Reconcile records from multiple sources into matched entities.\n\nMatches records across sources using identity field similarity,\nnormalizes to a unified vertical schema, and returns entity groups\nwith confidence scores.\n\nVerticals: product, job, real_estate, review",
- "operationId": "reconcile_endpoint_v1_reconcile_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_reconcile_endpoint_v1_reconcile_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Reconcile Endpoint V1 Reconcile Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/reconcile/schemas": {
- "get": {
- "tags": [
- "Reconciliation"
- ],
- "summary": "List supported reconciliation schemas",
- "description": "List all supported vertical schemas for entity reconciliation.",
- "operationId": "list_schemas_v1_reconcile_schemas_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Schemas V1 Reconcile Schemas Get"
- }
- }
- }
- }
- }
- }
- },
"/v1/auth/credentials": {
"get": {
"tags": [
@@ -4697,19 +249,85 @@
}
}
},
- "/v1/review/submit": {
+ "/health": {
+ "get": {
+ "tags": [
+ "Health"
+ ],
+ "summary": "Full health check with dependency status",
+ "description": "Comprehensive health check \u2014 probes Ollama, FlareSolverr, Redis.",
+ "operationId": "health_check_health_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {}
+ }
+ }
+ }
+ }
+ }
+ },
+ "/live": {
+ "get": {
+ "tags": [
+ "Health"
+ ],
+ "summary": "Kubernetes liveness probe",
+ "description": "Simple liveness \u2014 always returns 200 if the process is running.",
+ "operationId": "live_live_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": {
+ "type": "string"
+ },
+ "type": "object",
+ "title": "Response Live Live Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/ready": {
+ "get": {
+ "tags": [
+ "Health"
+ ],
+ "summary": "Kubernetes readiness probe",
+ "description": "Readiness \u2014 checks critical dependencies.",
+ "operationId": "ready_ready_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {}
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/scrape": {
"post": {
"tags": [
- "Review"
+ "Scraping"
],
- "summary": "Submit extracted data for human review",
- "description": "Submit extracted data for human review before delivery.\n\nUse this when extraction confidence is low or anomalies were detected.\nData will be held in the review queue until approved or rejected.",
- "operationId": "review_submit_v1_review_submit_post",
+ "summary": "Scrape a single URL",
+ "description": "Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.",
+ "operationId": "scrape_v1_scrape_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Body_review_submit_v1_review_submit_post"
+ "$ref": "#/components/schemas/ScrapeRequest"
}
}
},
@@ -4723,7 +341,7 @@
"schema": {
"additionalProperties": true,
"type": "object",
- "title": "Response Review Submit V1 Review Submit Post"
+ "title": "Response Scrape V1 Scrape Post"
}
}
}
@@ -4741,33 +359,24 @@
}
}
},
- "/v1/review/{review_id}/approve": {
+ "/v1/detect-block": {
"post": {
"tags": [
- "Review"
- ],
- "summary": "Approve a review item",
- "description": "Approve a review item, allowing data to proceed to delivery.",
- "operationId": "review_approve_v1_review__review_id__approve_post",
- "parameters": [
- {
- "name": "review_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Review Id"
- }
- }
+ "Scraping"
],
+ "summary": "Detect if a site is blocking the scraper",
+ "description": "Detect what kind of anti-bot protection a site is using.\n\nReturns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.\nUseful for debugging scraping issues.",
+ "operationId": "detect_block_v1_detect_block_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Body_review_approve_v1_review__review_id__approve_post"
+ "type": "string",
+ "title": "Url"
}
}
- }
+ },
+ "required": true
},
"responses": {
"200": {
@@ -4775,9 +384,9 @@
"content": {
"application/json": {
"schema": {
- "type": "object",
"additionalProperties": true,
- "title": "Response Review Approve V1 Review Review Id Approve Post"
+ "type": "object",
+ "title": "Response Detect Block V1 Detect Block Post"
}
}
}
@@ -4795,33 +404,23 @@
}
}
},
- "/v1/review/{review_id}/reject": {
+ "/v1/capture/lazy": {
"post": {
"tags": [
- "Review"
- ],
- "summary": "Reject a review item",
- "description": "Reject a review item, blocking data delivery.",
- "operationId": "review_reject_v1_review__review_id__reject_post",
- "parameters": [
- {
- "name": "review_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Review Id"
- }
- }
+ "Scraping"
],
+ "summary": "Detect and handle lazy-loaded content",
+ "description": "Detect lazy loading and infinite scroll patterns on a page.\n\nOptionally generate JS to auto-scroll and load all content.",
+ "operationId": "detect_lazy_content_v1_capture_lazy_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Body_review_reject_v1_review__review_id__reject_post"
+ "$ref": "#/components/schemas/Body_detect_lazy_content_v1_capture_lazy_post"
}
}
- }
+ },
+ "required": true
},
"responses": {
"200": {
@@ -4829,9 +428,9 @@
"content": {
"application/json": {
"schema": {
- "type": "object",
"additionalProperties": true,
- "title": "Response Review Reject V1 Review Review Id Reject Post"
+ "type": "object",
+ "title": "Response Detect Lazy Content V1 Capture Lazy Post"
}
}
}
@@ -4849,714 +448,111 @@
}
}
},
- "/v1/reviews": {
- "get": {
+ "/v1/crawl": {
+ "post": {
"tags": [
- "Review"
+ "Scraping"
],
- "summary": "List reviews in the queue",
- "description": "List reviews, optionally filtered by status (pending/approved/rejected).",
- "operationId": "list_reviews_v1_reviews_get",
+ "summary": "Crawl multiple pages from a URL",
+ "description": "Crawl multiple pages from a URL. Supports async webhooks.",
+ "operationId": "crawl_v1_crawl_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/CrawlRequest"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Crawl V1 Crawl Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/map": {
+ "post": {
+ "tags": [
+ "Scraping"
+ ],
+ "summary": "Discover URLs on a site",
+ "description": "Discover URLs on a site.",
+ "operationId": "map_pages_v1_map_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/MapRequest"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Map Pages V1 Map Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/batch": {
+ "post": {
+ "tags": [
+ "Scraping"
+ ],
+ "summary": "Scrape multiple URLs in parallel",
+ "description": "Scrape multiple URLs in parallel. Firecrawl charges extra for batch.",
+ "operationId": "batch_scrape_v1_batch_post",
"parameters": [
{
- "name": "status",
+ "name": "timeout",
"in": "query",
"required": false,
"schema": {
- "anyOf": [
- {
- "type": "string"
- },
- {
- "type": "null"
- }
- ],
- "title": "Status"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response List Reviews V1 Reviews Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/review/{review_id}": {
- "get": {
- "tags": [
- "Review"
- ],
- "summary": "Get review details",
- "description": "Get full details of a review item including the data payload.",
- "operationId": "get_review_v1_review__review_id__get",
- "parameters": [
- {
- "name": "review_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Review Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Review V1 Review Review Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/extract-with-review": {
- "post": {
- "tags": [
- "Review"
- ],
- "summary": "Extract with automatic human review routing",
- "description": "Extract data with automatic quality check and human review routing.\n\nHigh-confidence results are auto-approved.\nLow-confidence results are auto-rejected.\nMedium-confidence results go to the human review queue with Slack notification.",
- "operationId": "extract_with_review_v1_extract_with_review_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_extract_with_review_v1_extract_with_review_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Extract With Review V1 Extract With Review Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/commerce/sync": {
- "post": {
- "tags": [
- "Commerce"
- ],
- "summary": "Sync products to WooCommerce or Shopify",
- "description": "Sync scraped products to your e-commerce platform.\n\nPlatforms:\n- woocommerce: credentials = {\"wp_url\": \"...\", \"consumer_key\": \"...\", \"consumer_secret\": \"...\"}\n- shopify: credentials = {\"shop_url\": \"...\", \"access_token\": \"...\"}\n\nProducts are imported as drafts for review before publishing.",
- "operationId": "sync_commerce_v1_commerce_sync_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_sync_commerce_v1_commerce_sync_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Sync Commerce V1 Commerce Sync Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/commerce/platforms": {
- "get": {
- "tags": [
- "Commerce"
- ],
- "summary": "List supported commerce platforms",
- "description": "List supported e-commerce platforms and their credential requirements.",
- "operationId": "list_commerce_platforms_v1_commerce_platforms_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Commerce Platforms V1 Commerce Platforms Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/crm/sync": {
- "post": {
- "tags": [
- "CRM"
- ],
- "summary": "Sync scraped data to CRM",
- "description": "Sync scraped data to your CRM.\n\nPlatforms:\n- salesforce: credentials={\"instance_url\": \"...\", \"access_token\": \"...\"}\n- hubspot: credentials={\"api_key\": \"...\"}\n- pipedrive: credentials={\"api_token\": \"...\", \"domain\": \"...\"}\n- close: credentials={\"api_key\": \"...\"}\n\nObject types vary by platform:\n- Salesforce: Lead, Contact, Account, Opportunity\n- HubSpot: contacts, companies, deals\n- Pipedrive: person, organization, deal, lead\n- Close: lead, contact",
- "operationId": "sync_crm_v1_crm_sync_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_sync_crm_v1_crm_sync_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Sync Crm V1 Crm Sync Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/crm/platforms": {
- "get": {
- "tags": [
- "CRM"
- ],
- "summary": "List supported CRM platforms",
- "description": "List supported CRM platforms and their credential requirements.",
- "operationId": "list_crm_platforms_v1_crm_platforms_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Crm Platforms V1 Crm Platforms Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipelines/steps": {
- "get": {
- "tags": [
- "Pipelines"
- ],
- "summary": "List all available pipeline step types",
- "description": "List all available step types for the visual pipeline builder.\n\nEach step type includes: name, icon, description, required inputs with types,\nand expected outputs. A UI renders these as drag-and-drop blocks.",
- "operationId": "list_step_types_v1_pipelines_steps_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Step Types V1 Pipelines Steps Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipelines/validate": {
- "post": {
- "tags": [
- "Pipelines"
- ],
- "summary": "Validate a pipeline definition",
- "description": "Validate a pipeline definition for correctness.",
- "operationId": "validate_pipeline_endpoint_v1_pipelines_validate_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Pipeline"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Validate Pipeline Endpoint V1 Pipelines Validate Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipelines/run": {
- "post": {
- "tags": [
- "Pipelines"
- ],
- "summary": "Execute a pipeline",
- "description": "Execute a pipeline definition.\n\nSteps run sequentially. Each step's output is available as\n{{step_id.output_key}} in subsequent step templates.\n\nExample pipeline:\n{\n \"name\": \"Monitor Competitor Pricing\",\n \"steps\": [\n {\"id\": \"scrape_amazon\", \"type\": \"scrape\", \"inputs\": {\"url\": \"https://amazon.com/product\"}},\n {\"id\": \"extract\", \"type\": \"extract_css\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"schema\": {...}}},\n {\"id\": \"quality\", \"type\": \"quality_check\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"data\": \"{{extract.items}}\"}},\n {\"id\": \"notify\", \"type\": \"send_slack\", \"inputs\": {\"webhook_url\": \"https://hooks.slack.com/...\", \"message\": \"{{quality.quality_score}}\"}},\n ]\n}",
- "operationId": "run_pipeline_endpoint_v1_pipelines_run_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_run_pipeline_endpoint_v1_pipelines_run_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Run Pipeline Endpoint V1 Pipelines Run Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipelines/save": {
- "post": {
- "tags": [
- "Pipelines"
- ],
- "summary": "Save a pipeline definition",
- "description": "Save a pipeline definition for later use.",
- "operationId": "save_pipeline_endpoint_v1_pipelines_save_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Pipeline"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Save Pipeline Endpoint V1 Pipelines Save Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipelines": {
- "get": {
- "tags": [
- "Pipelines"
- ],
- "summary": "List saved pipelines",
- "description": "List all saved pipeline definitions.",
- "operationId": "list_pipelines_endpoint_v1_pipelines_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Pipelines Endpoint V1 Pipelines Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/pipelines/{pipeline_id}": {
- "get": {
- "tags": [
- "Pipelines"
- ],
- "summary": "Get a saved pipeline",
- "description": "Get a saved pipeline definition by ID.",
- "operationId": "get_pipeline_endpoint_v1_pipelines__pipeline_id__get",
- "parameters": [
- {
- "name": "pipeline_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Pipeline Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Pipeline Endpoint V1 Pipelines Pipeline Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- },
- "delete": {
- "tags": [
- "Pipelines"
- ],
- "summary": "Delete a saved pipeline",
- "description": "Delete a saved pipeline definition.",
- "operationId": "delete_pipeline_endpoint_v1_pipelines__pipeline_id__delete",
- "parameters": [
- {
- "name": "pipeline_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Pipeline Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Delete Pipeline Endpoint V1 Pipelines Pipeline Id Delete"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/agency/create": {
- "post": {
- "tags": [
- "Agency"
- ],
- "summary": "Create a white-label agency profile",
- "description": "Create a white-label agency profile for reselling Pry.\n\nAgencies get:\n- Custom branding (colors, logo, domain)\n- Client management with sub-accounts\n- Usage analytics and quota management\n- API key management for each client",
- "operationId": "create_agency_endpoint_v1_agency_create_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_create_agency_endpoint_v1_agency_create_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Create Agency Endpoint V1 Agency Create Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/agency/{agency_id}": {
- "get": {
- "tags": [
- "Agency"
- ],
- "summary": "Get agency profile",
- "description": "Get agency profile details.",
- "operationId": "get_agency_endpoint_v1_agency__agency_id__get",
- "parameters": [
- {
- "name": "agency_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Agency Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Agency Endpoint V1 Agency Agency Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/agency/{agency_id}/branding": {
- "put": {
- "tags": [
- "Agency"
- ],
- "summary": "Update agency branding",
- "description": "Update white-label branding for an agency.",
- "operationId": "update_branding_v1_agency__agency_id__branding_put",
- "parameters": [
- {
- "name": "agency_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Agency Id"
- }
- }
- ],
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_update_branding_v1_agency__agency_id__branding_put"
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Update Branding V1 Agency Agency Id Branding Put"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/agency/{agency_id}/clients": {
- "post": {
- "tags": [
- "Agency"
- ],
- "summary": "Create a client sub-account",
- "description": "Create a client sub-account under an agency.\n\nEach client gets their own API key and usage quota.",
- "operationId": "create_client_endpoint_v1_agency__agency_id__clients_post",
- "parameters": [
- {
- "name": "agency_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Agency Id"
+ "type": "integer",
+ "default": 30,
+ "title": "Timeout"
}
}
],
@@ -5565,7 +561,11 @@
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Body_create_client_endpoint_v1_agency__agency_id__clients_post"
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "title": "Urls"
}
}
}
@@ -5578,474 +578,7 @@
"schema": {
"type": "object",
"additionalProperties": true,
- "title": "Response Create Client Endpoint V1 Agency Agency Id Clients Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- },
- "get": {
- "tags": [
- "Agency"
- ],
- "summary": "List agency clients",
- "description": "List all clients under an agency.",
- "operationId": "list_clients_endpoint_v1_agency__agency_id__clients_get",
- "parameters": [
- {
- "name": "agency_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Agency Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response List Clients Endpoint V1 Agency Agency Id Clients Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/agency/{agency_id}/analytics": {
- "get": {
- "tags": [
- "Agency"
- ],
- "summary": "Get agency usage analytics",
- "description": "Get aggregate usage analytics for an agency.",
- "operationId": "get_analytics_v1_agency__agency_id__analytics_get",
- "parameters": [
- {
- "name": "agency_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Agency Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Analytics V1 Agency Agency Id Analytics Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/client/{client_id}/quota": {
- "get": {
- "tags": [
- "Agency"
- ],
- "summary": "Check client quota usage",
- "description": "Check a client's current quota usage and remaining capacity.",
- "operationId": "check_quota_v1_client__client_id__quota_get",
- "parameters": [
- {
- "name": "client_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Client Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Check Quota V1 Client Client Id Quota Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/seo/analyze": {
- "post": {
- "tags": [
- "SEO"
- ],
- "summary": "Analyze SEO elements from a URL",
- "description": "Analyze all SEO elements from a URL.\n\nReturns: title, meta description, keywords, headings (H1/H2),\ncanonical, OG tags, Twitter cards, word count, link counts,\nschema markup, hreflang tags.",
- "operationId": "seo_analyze_v1_seo_analyze_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Seo Analyze V1 Seo Analyze Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/seo/track": {
- "post": {
- "tags": [
- "SEO"
- ],
- "summary": "Track SEO changes since last scan",
- "description": "Track SEO changes since the last scan of this URL.\n\nCompares current SEO elements to previous snapshot and reports\nwhat changed (title, description, headings, etc.).",
- "operationId": "seo_track_v1_seo_track_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Seo Track V1 Seo Track Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/seo/keywords": {
- "post": {
- "tags": [
- "SEO"
- ],
- "summary": "Analyze keyword presence in URL content",
- "description": "Analyze which keywords a URL targets.\n\nChecks each keyword for:\n- Presence in title tag\n- Presence in H1 headings\n- Presence in meta description\n- Frequency in body content\n- Keyword density percentage",
- "operationId": "seo_keywords_v1_seo_keywords_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_seo_keywords_v1_seo_keywords_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Seo Keywords V1 Seo Keywords Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/reports/generate": {
- "post": {
- "tags": [
- "Reports"
- ],
- "summary": "Generate a white-label report from scraped data",
- "description": "Generate a white-label report from scraped data.\n\nReport types:\n- competitive_analysis: Competitor pricing and activity overview\n- price_monitor: Product price change tracking with visual indicators\n- seo_audit: SEO element analysis with change detection\n- content_tracker: Content change monitoring across pages\n\nBranding (optional): {\"agency_name\": \"...\", \"brand_color\": \"#hex\", \"logo_url\": \"...\"}",
- "operationId": "generate_report_endpoint_v1_reports_generate_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_generate_report_endpoint_v1_reports_generate_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Generate Report Endpoint V1 Reports Generate Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/reports": {
- "get": {
- "tags": [
- "Reports"
- ],
- "summary": "List generated reports",
- "description": "List all previously generated reports.",
- "operationId": "list_reports_endpoint_v1_reports_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Reports Endpoint V1 Reports Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/report/{report_id}": {
- "get": {
- "tags": [
- "Reports"
- ],
- "summary": "Get a generated report",
- "description": "Get the HTML content of a generated report.",
- "operationId": "get_report_v1_report__report_id__get",
- "parameters": [
- {
- "name": "report_id",
- "in": "path",
- "required": true,
- "schema": {
- "type": "string",
- "title": "Report Id"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "title": "Response Get Report V1 Report Report Id Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/enrich": {
- "post": {
- "tags": [
- "Enrichment"
- ],
- "summary": "Enrich scraped data with company info, tech stack, social profiles",
- "description": "Enrich a URL with supplemental business intelligence.\n\nReturns:\n- Tech stack detection (CMS, framework, CDN, analytics, payments)\n- Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.)\n- Company information (email, phone, address, founded year, team size)",
- "operationId": "enrich_data_v1_enrich_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_enrich_data_v1_enrich_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Enrich Data V1 Enrich Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/enrich/tech-stack": {
- "post": {
- "tags": [
- "Enrichment"
- ],
- "summary": "Detect technologies used on a website",
- "description": "Detect what technologies a website is built with.\n\nDetects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails),\nfrontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar),\npayments (Stripe, PayPal).",
- "operationId": "detect_tech_v1_enrich_tech_stack_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Url"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Detect Tech V1 Enrich Tech Stack Post"
+ "title": "Response Batch Scrape V1 Batch Post"
}
}
}
@@ -6176,9 +709,58 @@
}
}
},
+ "/v1/templates/batch": {
+ "post": {
+ "tags": [
+ "Templates"
+ ],
+ "summary": "Execute multiple templates/URLs in parallel",
+ "description": "Execute multiple scraper templates against multiple URLs in parallel.\n\nRuns up to 20 template executions concurrently. Each result preserves the\noriginal template_id and url for correlation.",
+ "operationId": "batch_execute_v1_templates_batch_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "items": {
+ "$ref": "#/components/schemas/BatchItem"
+ },
+ "type": "array",
+ "title": "Body"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Batch Execute V1 Templates Batch Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/v1/ai/gpt-manifest": {
"get": {
"tags": [
+ "AI",
"AI"
],
"summary": "Get GPT Action manifest for ChatGPT",
@@ -6199,6 +781,7 @@
"/v1/ai/mcp-config": {
"get": {
"tags": [
+ "AI",
"AI"
],
"summary": "Get MCP server config for Claude/Cursor",
@@ -6216,689 +799,10 @@
}
}
},
- "/v1/referrals/catalog": {
- "get": {
- "tags": [
- "Referrals"
- ],
- "summary": "Get all available referral/affiliate programs",
- "description": "List all referral programs Pry supports.\n\n60+ providers across categories: LLM, hosting, domains, CDN, email,\nmonitoring, proxies, voice, media, devtools, search, CAPTCHA.",
- "operationId": "get_referral_catalog_v1_referrals_catalog_get",
- "parameters": [
- {
- "name": "category",
- "in": "query",
- "required": false,
- "schema": {
- "type": "string",
- "default": "",
- "title": "Category"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Referral Catalog V1 Referrals Catalog Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/referrals/stats": {
- "get": {
- "tags": [
- "Referrals"
- ],
- "summary": "Get referral click and conversion stats",
- "description": "Get referral tracking statistics for the last N days.",
- "operationId": "get_referral_stats_v1_referrals_stats_get",
- "parameters": [
- {
- "name": "days_back",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 30,
- "title": "Days Back"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Get Referral Stats V1 Referrals Stats Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/referrals/click": {
- "post": {
- "tags": [
- "Referrals"
- ],
- "summary": "Record a referral link click",
- "description": "Record when a user clicks a referral link. Returns tracking ID.",
- "operationId": "record_referral_click_v1_referrals_click_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_record_referral_click_v1_referrals_click_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Record Referral Click V1 Referrals Click Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/referrals/convert": {
- "post": {
- "tags": [
- "Referrals"
- ],
- "summary": "Record a referral conversion",
- "description": "Record that a referral click resulted in a conversion.",
- "operationId": "record_referral_conversion_v1_referrals_convert_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_record_referral_conversion_v1_referrals_convert_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Record Referral Conversion V1 Referrals Convert Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/x402/pricing": {
- "get": {
- "tags": [
- "x402"
- ],
- "summary": "Get x402 pricing for all paid operations",
- "description": "Get the price list for pay-per-scrape operations.",
- "operationId": "x402_pricing_v1_x402_pricing_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response X402 Pricing V1 X402 Pricing Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/x402/payment": {
- "post": {
- "tags": [
- "x402"
- ],
- "summary": "Create a x402 payment request",
- "description": "Create a x402 payment request for a paid operation.\n\nReturns payment details (wallet, amount, asset) for the client to pay.",
- "operationId": "x402_payment_request_v1_x402_payment_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_x402_payment_request_v1_x402_payment_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response X402 Payment Request V1 X402 Payment Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/x402/verify": {
- "post": {
- "tags": [
- "x402"
- ],
- "summary": "Verify a x402 payment was settled",
- "description": "Verify a x402 payment has been settled on-chain via facilitator router.",
- "operationId": "x402_verify_payment_v1_x402_verify_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_x402_verify_payment_v1_x402_verify_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response X402 Verify Payment V1 X402 Verify Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/x402/require-payment": {
- "post": {
- "tags": [
- "x402"
- ],
- "summary": "Generate a 402 Payment Required response",
- "description": "Generate a 402 Payment Required response for a paid endpoint.",
- "operationId": "x402_require_payment_v1_x402_require_payment_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_x402_require_payment_v1_x402_require_payment_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response X402 Require Payment V1 X402 Require Payment Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/x402/batch-payment": {
- "post": {
- "tags": [
- "x402"
- ],
- "summary": "Create a batch x402 payment",
- "description": "Create a single x402 payment covering multiple operations.\n\nReturns a PaymentRequired body with the combined amount. After paying,\nsubmit the tx to POST /v1/x402/batch-verify.",
- "operationId": "x402_batch_payment_v1_x402_batch_payment_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Payload"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response X402 Batch Payment V1 X402 Batch Payment Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/x402/batch-verify": {
- "post": {
- "tags": [
- "x402"
- ],
- "summary": "Verify a batch x402 payment",
- "description": "Verify the on-chain payment for a batch and mark it paid.",
- "operationId": "x402_batch_verify_v1_x402_batch_verify_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Payload"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response X402 Batch Verify V1 X402 Batch Verify Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/providers": {
- "get": {
- "tags": [
- "Proxy"
- ],
- "summary": "List all available proxy providers",
- "description": "List free + premium proxy providers with affiliate details.",
- "operationId": "list_proxy_providers_v1_proxy_providers_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response List Proxy Providers V1 Proxy Providers Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/signup": {
- "post": {
- "tags": [
- "Proxy"
- ],
- "summary": "Open affiliate signup link for a provider",
- "description": "Get the affiliate signup URL for a proxy provider.\n\nRecords a click for revenue tracking. The user can then sign up at that URL\nand come back to configure credentials.",
- "operationId": "proxy_signup_v1_proxy_signup_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Provider"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Proxy Signup V1 Proxy Signup Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/configure": {
- "post": {
- "tags": [
- "Proxy"
- ],
- "summary": "Configure proxy credentials",
- "description": "Configure credentials for a premium proxy provider.\n\nAfter signing up via /v1/proxy/signup, the user provides their credentials here.",
- "operationId": "proxy_configure_v1_proxy_configure_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/Body_proxy_configure_v1_proxy_configure_post"
- }
- }
- },
- "required": true
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Proxy Configure V1 Proxy Configure Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/test": {
- "get": {
- "tags": [
- "Proxy"
- ],
- "summary": "Test the active proxy",
- "description": "Test the currently configured proxy and return its public IP.",
- "operationId": "proxy_test_v1_proxy_test_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Proxy Test V1 Proxy Test Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/status": {
- "get": {
- "tags": [
- "Proxy"
- ],
- "summary": "Get current proxy status",
- "description": "Get current proxy configuration and available providers.",
- "operationId": "proxy_status_v1_proxy_status_get",
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Proxy Status V1 Proxy Status Get"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/recommend": {
- "post": {
- "tags": [
- "Proxy"
- ],
- "summary": "Get proxy recommendation after a block",
- "description": "After a scrape fails with anti-bot detection, get a recommendation\nfor which premium proxy provider to sign up with.",
- "operationId": "proxy_recommend_v1_proxy_recommend_post",
- "requestBody": {
- "content": {
- "application/json": {
- "schema": {
- "type": "string",
- "title": "Last Error",
- "default": ""
- }
- }
- }
- },
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "additionalProperties": true,
- "type": "object",
- "title": "Response Proxy Recommend V1 Proxy Recommend Post"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
- "/v1/proxy/clicks": {
- "get": {
- "tags": [
- "Proxy"
- ],
- "summary": "Get recent proxy referral clicks",
- "description": "Get recent proxy referral clicks for revenue tracking.",
- "operationId": "proxy_clicks_v1_proxy_clicks_get",
- "parameters": [
- {
- "name": "days_back",
- "in": "query",
- "required": false,
- "schema": {
- "type": "integer",
- "default": 30,
- "title": "Days Back"
- }
- }
- ],
- "responses": {
- "200": {
- "description": "Successful Response",
- "content": {
- "application/json": {
- "schema": {
- "type": "object",
- "additionalProperties": true,
- "title": "Response Proxy Clicks V1 Proxy Clicks Get"
- }
- }
- }
- },
- "422": {
- "description": "Validation Error",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/HTTPValidationError"
- }
- }
- }
- }
- }
- }
- },
"/v1/tls/impersonate": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Fetch a URL with TLS fingerprint impersonation",
@@ -6943,6 +847,7 @@
"/v1/tls/rotate": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Try multiple browser fingerprints until one succeeds",
@@ -6987,6 +892,7 @@
"/v1/camoufox/fetch": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Fetch with Camoufox anti-detection Firefox",
@@ -7031,6 +937,7 @@
"/v1/graphql/discover": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Discover GraphQL endpoints for a site",
@@ -7076,6 +983,7 @@
"/v1/graphql/introspect": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Run GraphQL introspection on an endpoint",
@@ -7121,6 +1029,7 @@
"/v1/graphql/query": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Execute a GraphQL query",
@@ -7165,6 +1074,7 @@
"/v1/schema/extract": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Extract Schema.org structured data from a page",
@@ -7210,6 +1120,7 @@
"/v1/schema/extract-html": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Extract Schema.org structured data from raw HTML",
@@ -7255,6 +1166,7 @@
"/v1/ws/scrape": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Scrape data from a WebSocket endpoint",
@@ -7299,6 +1211,7 @@
"/v1/sse/scrape": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Scrape data from a Server-Sent Events endpoint",
@@ -7343,6 +1256,7 @@
"/v1/cookies/warm": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Warm cookies for a domain by browsing legitimate pages",
@@ -7387,6 +1301,7 @@
"/v1/cookies/sessions": {
"get": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "List all warmed cookie sessions",
@@ -7410,6 +1325,7 @@
"/v1/pdf/extract": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Extract tables and text from a PDF",
@@ -7454,6 +1370,7 @@
"/v1/ocr/extract": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Extract text from an image using Tesseract",
@@ -7497,6 +1414,7 @@
"/v1/dedup/check": {
"post": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Check if content is a near-duplicate using SimHash",
@@ -7541,6 +1459,7 @@
"/v1/behavior/simulate": {
"get": {
"tags": [
+ "Advanced",
"Advanced"
],
"summary": "Generate human-like behavior patterns for testing",
@@ -7584,19 +1503,20 @@
}
}
},
- "/v1/x402/pay": {
+ "/v1/agency/create": {
"post": {
"tags": [
- "x402"
+ "Agency",
+ "Agency"
],
- "summary": "Process x402 payment and get access token",
- "description": "Process an x402 payment and return an access token.\n\nFlow:\n1. User gets 402 from a paid endpoint\n2. User sends USDC to the wallet in the 402 response\n3. User calls this endpoint with the tx_hash\n4. Pry verifies the transaction through the facilitator router\n5. Returns access token (payment_id) to use in X-Payment-Hash header",
- "operationId": "x402_pay_v1_x402_pay_post",
+ "summary": "Create a white-label agency profile",
+ "description": "Create a white-label agency profile for reselling Pry.\n\nAgencies get:\n- Custom branding (colors, logo, domain)\n- Client management with sub-accounts\n- Usage analytics and quota management\n- API key management for each client",
+ "operationId": "create_agency_endpoint_v1_agency_create_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
- "$ref": "#/components/schemas/Body_x402_pay_v1_x402_pay_post"
+ "$ref": "#/components/schemas/Body_create_agency_endpoint_v1_agency_create_post"
}
}
},
@@ -7610,7 +1530,2979 @@
"schema": {
"additionalProperties": true,
"type": "object",
- "title": "Response X402 Pay V1 X402 Pay Post"
+ "title": "Response Create Agency Endpoint V1 Agency Create Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/agency/{agency_id}": {
+ "get": {
+ "tags": [
+ "Agency",
+ "Agency"
+ ],
+ "summary": "Get agency profile",
+ "description": "Get agency profile details.",
+ "operationId": "get_agency_endpoint_v1_agency__agency_id__get",
+ "parameters": [
+ {
+ "name": "agency_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Agency Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Agency Endpoint V1 Agency Agency Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/agency/{agency_id}/branding": {
+ "put": {
+ "tags": [
+ "Agency",
+ "Agency"
+ ],
+ "summary": "Update agency branding",
+ "description": "Update white-label branding for an agency.",
+ "operationId": "update_branding_v1_agency__agency_id__branding_put",
+ "parameters": [
+ {
+ "name": "agency_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Agency Id"
+ }
+ }
+ ],
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_update_branding_v1_agency__agency_id__branding_put"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Update Branding V1 Agency Agency Id Branding Put"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/agency/{agency_id}/clients": {
+ "post": {
+ "tags": [
+ "Agency",
+ "Agency"
+ ],
+ "summary": "Create a client sub-account",
+ "description": "Create a client sub-account under an agency.\n\nEach client gets their own API key and usage quota.",
+ "operationId": "create_client_endpoint_v1_agency__agency_id__clients_post",
+ "parameters": [
+ {
+ "name": "agency_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Agency Id"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_create_client_endpoint_v1_agency__agency_id__clients_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Create Client Endpoint V1 Agency Agency Id Clients Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "get": {
+ "tags": [
+ "Agency",
+ "Agency"
+ ],
+ "summary": "List agency clients",
+ "description": "List all clients under an agency.",
+ "operationId": "list_clients_endpoint_v1_agency__agency_id__clients_get",
+ "parameters": [
+ {
+ "name": "agency_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Agency Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response List Clients Endpoint V1 Agency Agency Id Clients Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/agency/{agency_id}/analytics": {
+ "get": {
+ "tags": [
+ "Agency",
+ "Agency"
+ ],
+ "summary": "Get agency usage analytics",
+ "description": "Get aggregate usage analytics for an agency.",
+ "operationId": "get_analytics_v1_agency__agency_id__analytics_get",
+ "parameters": [
+ {
+ "name": "agency_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Agency Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Analytics V1 Agency Agency Id Analytics Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/client/{client_id}/quota": {
+ "get": {
+ "tags": [
+ "Agency",
+ "Agency"
+ ],
+ "summary": "Check client quota usage",
+ "description": "Check a client's current quota usage and remaining capacity.",
+ "operationId": "check_quota_v1_client__client_id__quota_get",
+ "parameters": [
+ {
+ "name": "client_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Client Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Check Quota V1 Client Client Id Quota Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/alert/send": {
+ "post": {
+ "tags": [
+ "Alerts",
+ "Alerts"
+ ],
+ "summary": "Send an alert to any channel",
+ "description": "Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email.\n\nConfig varies by channel:\n- slack: {\"webhook_url\": \"...\"}\n- discord: {\"webhook_url\": \"...\"}\n- teams: {\"webhook_url\": \"...\"}\n- telegram: {\"bot_token\": \"...\", \"chat_id\": \"...\"}\n- sms: {\"phone\": \"+1234567890\", \"twilio_sid\": \"...\", \"twilio_token\": \"...\", \"twilio_from\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"smtp_host\": \"...\", \"smtp_user\": \"...\"}",
+ "operationId": "send_alert_endpoint_v1_alert_send_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_send_alert_endpoint_v1_alert_send_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Send Alert Endpoint V1 Alert Send Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/alert/channels": {
+ "get": {
+ "tags": [
+ "Alerts",
+ "Alerts"
+ ],
+ "summary": "List supported alert channels",
+ "description": "List all supported alert channels and their config requirements.",
+ "operationId": "list_channels_v1_alert_channels_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Channels V1 Alert Channels Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/compare": {
+ "post": {
+ "tags": [
+ "Analysis",
+ "Analysis"
+ ],
+ "summary": "Compare content of two URLs",
+ "description": "Scrape two URLs and compare their content. Shows additions, deletions, changes.",
+ "operationId": "compare_v1_compare_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_compare_v1_compare_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Compare V1 Compare Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/watch": {
+ "post": {
+ "tags": [
+ "Analysis",
+ "Analysis"
+ ],
+ "summary": "Watch a page for changes",
+ "description": "Watch a page for changes. Accepts a webhook URL for notification.\nFirst call registers the watch, subsequent calls compare and notify.\nFirecrawl charges $99/mo for this feature.",
+ "operationId": "watch_page_v1_watch_post",
+ "parameters": [
+ {
+ "name": "interval",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 3600,
+ "title": "Interval"
+ }
+ },
+ {
+ "name": "selector",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "",
+ "title": "Selector"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_watch_page_v1_watch_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Watch Page V1 Watch Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/summarize": {
+ "post": {
+ "tags": [
+ "Analysis",
+ "Analysis"
+ ],
+ "summary": "AI summarize scraped content",
+ "description": "Scrape + AI summarize using local Ollama. Free, private.",
+ "operationId": "summarize_v1_summarize_post",
+ "parameters": [
+ {
+ "name": "max_words",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 100,
+ "title": "Max Words"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Summarize V1 Summarize Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/diff": {
+ "post": {
+ "tags": [
+ "Analysis",
+ "Analysis"
+ ],
+ "summary": "Track page changes over time",
+ "description": "Track page changes over time. Returns diff from last scrape.",
+ "operationId": "diff_v1_diff_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_diff_v1_diff_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Diff V1 Diff Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/categorize": {
+ "post": {
+ "tags": [
+ "Analysis",
+ "Analysis"
+ ],
+ "summary": "AI-categorize scraped content",
+ "description": "AI-categorize scraped content into topic tags.",
+ "operationId": "categorize_v1_categorize_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Categorize V1 Categorize Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/suggest": {
+ "post": {
+ "tags": [
+ "Analysis",
+ "Analysis"
+ ],
+ "summary": "AI-suggest schema fields for a URL",
+ "operationId": "suggest_schema_v1_suggest_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Data"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Suggest Schema V1 Suggest Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/automate": {
+ "post": {
+ "tags": [
+ "Automation",
+ "Automation"
+ ],
+ "summary": "Execute browser automation steps",
+ "description": "Execute browser automation steps. Sessions persist for login flows.",
+ "operationId": "automate_v1_automate_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/AutomateRequest"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Automate V1 Automate Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/screenshot": {
+ "post": {
+ "tags": [
+ "Automation",
+ "Automation"
+ ],
+ "summary": "Take a screenshot of a URL",
+ "description": "Take a screenshot of a URL. Returns base64 PNG.",
+ "operationId": "screenshot_v1_screenshot_post",
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Session Id"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_screenshot_v1_screenshot_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Screenshot V1 Screenshot Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/batch-file": {
+ "post": {
+ "tags": [
+ "Batch",
+ "Batch"
+ ],
+ "summary": "Batch scrape URLs from a file with templates",
+ "operationId": "batch_from_file_v1_batch_file_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/BatchFileRequest"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Batch From File V1 Batch File Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/crm/sync": {
+ "post": {
+ "tags": [
+ "CRM",
+ "CRM"
+ ],
+ "summary": "Sync scraped data to CRM",
+ "description": "Sync scraped data to your CRM.\n\nPlatforms:\n- salesforce: credentials={\"instance_url\": \"...\", \"access_token\": \"...\"}\n- hubspot: credentials={\"api_key\": \"...\"}\n- pipedrive: credentials={\"api_token\": \"...\", \"domain\": \"...\"}\n- close: credentials={\"api_key\": \"...\"}\n\nObject types vary by platform:\n- Salesforce: Lead, Contact, Account, Opportunity\n- HubSpot: contacts, companies, deals\n- Pipedrive: person, organization, deal, lead\n- Close: lead, contact",
+ "operationId": "sync_crm_v1_crm_sync_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_sync_crm_v1_crm_sync_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Sync Crm V1 Crm Sync Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/crm/platforms": {
+ "get": {
+ "tags": [
+ "CRM",
+ "CRM"
+ ],
+ "summary": "List supported CRM platforms",
+ "description": "List supported CRM platforms and their credential requirements.",
+ "operationId": "list_crm_platforms_v1_crm_platforms_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Crm Platforms V1 Crm Platforms Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/breaker/status": {
+ "post": {
+ "tags": [
+ "Circuit Breaker",
+ "Circuit Breaker"
+ ],
+ "summary": "Get circuit breaker status",
+ "operationId": "breaker_status_v1_breaker_status_post",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Breaker Status V1 Breaker Status Post"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/breaker/reset": {
+ "post": {
+ "tags": [
+ "Circuit Breaker",
+ "Circuit Breaker"
+ ],
+ "summary": "Reset circuit breaker for a domain",
+ "operationId": "breaker_reset_v1_breaker_reset_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Domain",
+ "default": ""
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Breaker Reset V1 Breaker Reset Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/commerce/sync": {
+ "post": {
+ "tags": [
+ "Commerce",
+ "Commerce"
+ ],
+ "summary": "Sync products to WooCommerce or Shopify",
+ "description": "Sync scraped products to your e-commerce platform.\n\nPlatforms:\n- woocommerce: credentials = {\"wp_url\": \"...\", \"consumer_key\": \"...\", \"consumer_secret\": \"...\"}\n- shopify: credentials = {\"shop_url\": \"...\", \"access_token\": \"...\"}\n\nProducts are imported as drafts for review before publishing.",
+ "operationId": "sync_commerce_v1_commerce_sync_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_sync_commerce_v1_commerce_sync_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Sync Commerce V1 Commerce Sync Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/commerce/platforms": {
+ "get": {
+ "tags": [
+ "Commerce",
+ "Commerce"
+ ],
+ "summary": "List supported commerce platforms",
+ "description": "List supported e-commerce platforms and their credential requirements.",
+ "operationId": "list_commerce_platforms_v1_commerce_platforms_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Commerce Platforms V1 Commerce Platforms Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/compliance/check": {
+ "get": {
+ "tags": [
+ "Compliance",
+ "Compliance"
+ ],
+ "summary": "Get compliance check documentation",
+ "description": "Get information about the compliance check endpoint.",
+ "operationId": "compliance_docs_v1_compliance_check_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Compliance Docs V1 Compliance Check Get"
+ }
+ }
+ }
+ }
+ }
+ },
+ "post": {
+ "tags": [
+ "Compliance",
+ "Compliance"
+ ],
+ "summary": "Run full compliance check on a URL",
+ "description": "Run a full legal compliance check on a target URL.\n\nAnalyzes:\n- robots.txt crawl permissions\n- Terms of Service classification (restrictive/permissive/moderate)\n- Jurisdiction detection (GDPR, CCPA, LGPD)\n- Sensitive data detection (PII, financial, health, contact)\n\nReturns a green/yellow/red risk score with recommendations.",
+ "operationId": "compliance_check_v1_compliance_check_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Compliance Check V1 Compliance Check Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/config": {
+ "get": {
+ "tags": [
+ "Config",
+ "Config"
+ ],
+ "summary": "Get current Pry configuration",
+ "description": "Get current Pry configuration.",
+ "operationId": "get_config_v1_config_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Get Config V1 Config Get"
+ }
+ }
+ }
+ }
+ }
+ },
+ "post": {
+ "tags": [
+ "Config",
+ "Config"
+ ],
+ "summary": "Update Pry configuration at runtime",
+ "description": "Update Pry configuration at runtime.",
+ "operationId": "update_config_v1_config_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Updates"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Update Config V1 Config Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/config/profile/tor": {
+ "post": {
+ "tags": [
+ "Config",
+ "Config"
+ ],
+ "summary": "Enable Tor routing for all requests",
+ "description": "Enable Tor routing for all requests.",
+ "operationId": "enable_tor_v1_config_profile_tor_post",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Enable Tor V1 Config Profile Tor Post"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/costing/dashboard": {
+ "get": {
+ "tags": [
+ "Costing",
+ "Costing"
+ ],
+ "summary": "Get cost analytics dashboard",
+ "description": "Get the full cost analytics dashboard.\n\nIncludes:\n- Monthly spend breakdown by operation type\n- Projected end-of-month cost\n- Daily spend for last 7 days\n- Cache efficiency metrics\n- Smart schedule recommendations",
+ "operationId": "cost_dashboard_v1_costing_dashboard_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Cost Dashboard V1 Costing Dashboard Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/costing/usage": {
+ "get": {
+ "tags": [
+ "Costing",
+ "Costing"
+ ],
+ "summary": "Get usage breakdown",
+ "description": "Get detailed usage breakdown for a specific month.",
+ "operationId": "cost_usage_v1_costing_usage_get",
+ "parameters": [
+ {
+ "name": "year",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Year"
+ }
+ },
+ {
+ "name": "month",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Month"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Cost Usage V1 Costing Usage Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/costing/record": {
+ "post": {
+ "tags": [
+ "Costing",
+ "Costing"
+ ],
+ "summary": "Record a usage event",
+ "description": "Record a usage event for cost tracking.\n\nOperations: scrape_direct, scrape_flaresolverr, scrape_playwright,\ncrawl_page, llm_call, vision_call, extraction_css, bandwidth_mb",
+ "operationId": "record_usage_endpoint_v1_costing_record_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_record_usage_endpoint_v1_costing_record_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Record Usage Endpoint V1 Costing Record Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/costing/costs": {
+ "post": {
+ "tags": [
+ "Costing",
+ "Costing"
+ ],
+ "summary": "Update per-operation cost table",
+ "description": "Update the per-operation cost table with custom prices.",
+ "operationId": "update_costs_v1_costing_costs_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": {
+ "type": "number"
+ },
+ "type": "object",
+ "title": "Costs"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Update Costs V1 Costing Costs Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/dashboard": {
+ "get": {
+ "tags": [
+ "Dashboard",
+ "Dashboard"
+ ],
+ "summary": "Pry health and performance dashboard",
+ "operationId": "dashboard_dashboard_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {}
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/email/scrape": {
+ "post": {
+ "tags": [
+ "Email",
+ "Email"
+ ],
+ "summary": "Extract data from an email (subject + body)",
+ "description": "Extract structured data from an email subject and body.\n\nAuto-classifies as: order_confirmation, invoice, receipt,\nshipping_notification, subscription, or other.\n\nExtracts: order numbers, amounts, tracking numbers, dates, addresses.",
+ "operationId": "scrape_email_v1_email_scrape_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_scrape_email_v1_email_scrape_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Scrape Email V1 Email Scrape Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/email/gmail": {
+ "post": {
+ "tags": [
+ "Email",
+ "Email"
+ ],
+ "summary": "Fetch and extract data from Gmail inbox",
+ "description": "Connect to Gmail and extract structured data from emails.\n\nRequires a Gmail OAuth2 access token with scope:\nhttps://www.googleapis.com/auth/gmail.readonly\n\nExtracts order confirmations, invoices, receipts, shipping notifications.",
+ "operationId": "fetch_gmail_v1_email_gmail_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_fetch_gmail_v1_email_gmail_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Fetch Gmail V1 Email Gmail Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/email/outlook": {
+ "post": {
+ "tags": [
+ "Email",
+ "Email"
+ ],
+ "summary": "Fetch and extract data from Outlook inbox",
+ "description": "Connect to Outlook/Office 365 and extract structured data from emails.\n\nRequires a Microsoft Graph OAuth2 access token with scope:\nhttps://graph.microsoft.com/Mail.Read\n\nExtracts order confirmations, invoices, receipts, shipping notifications.",
+ "operationId": "fetch_outlook_v1_email_outlook_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_fetch_outlook_v1_email_outlook_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Fetch Outlook V1 Email Outlook Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/enrich": {
+ "post": {
+ "tags": [
+ "Enrichment",
+ "Enrichment"
+ ],
+ "summary": "Enrich scraped data with company info, tech stack, social profiles",
+ "description": "Enrich a URL with supplemental business intelligence.\n\nReturns:\n- Tech stack detection (CMS, framework, CDN, analytics, payments)\n- Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.)\n- Company information (email, phone, address, founded year, team size)",
+ "operationId": "enrich_data_v1_enrich_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_enrich_data_v1_enrich_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Enrich Data V1 Enrich Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/enrich/tech-stack": {
+ "post": {
+ "tags": [
+ "Enrichment",
+ "Enrichment"
+ ],
+ "summary": "Detect technologies used on a website",
+ "description": "Detect what technologies a website is built with.\n\nDetects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails),\nfrontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar),\npayments (Stripe, PayPal).",
+ "operationId": "detect_tech_v1_enrich_tech_stack_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Detect Tech V1 Enrich Tech Stack Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/run": {
+ "post": {
+ "tags": [
+ "Execute",
+ "Execute"
+ ],
+ "summary": "Execute a Pryfile",
+ "operationId": "run_pryfile_v1_run_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Path",
+ "default": "pry.yml"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Run Pryfile V1 Run Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/export": {
+ "post": {
+ "tags": [
+ "Export",
+ "Export"
+ ],
+ "summary": "Export scraped content in multiple formats",
+ "description": "Export scraped content in multiple formats: json, csv, txt, rss.\nFirecrawl only does markdown.",
+ "operationId": "export_data_v1_export_post",
+ "parameters": [
+ {
+ "name": "format",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "json",
+ "title": "Format"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Export Data V1 Export Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/extract-table": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Extract HTML tables as structured data",
+ "description": "Extract HTML tables from a page as structured data.\nFirecrawl doesn't support table extraction at all.",
+ "operationId": "extract_table_v1_extract_table_post",
+ "parameters": [
+ {
+ "name": "table_index",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 0,
+ "title": "Table Index"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Extract Table V1 Extract Table Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/links": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Analyze links on a page",
+ "description": "Analyze all links on a page \u2014 internal, external, broken, social.\nFirecrawl only has basic map functionality.",
+ "operationId": "analyze_links_v1_links_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Analyze Links V1 Links Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/seo": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "SEO analysis of a page",
+ "description": "SEO analysis of a page: title, description, headings, images, keywords, readability.\nFirecrawl has zero SEO features.",
+ "operationId": "analyze_seo_v1_seo_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Analyze Seo V1 Seo Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/schema": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Extract Schema.org/JSON-LD structured data",
+ "description": "Extract Schema.org/JSON-LD structured data from a page.",
+ "operationId": "extract_schema_v1_schema_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Extract Schema V1 Schema Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/emails": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Find email addresses on a page",
+ "description": "Find all email addresses on a page.",
+ "operationId": "find_emails_v1_emails_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Find Emails V1 Emails Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/extract": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Extract structured fields from a URL",
+ "operationId": "extract_stable_v1_extract_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Data"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Extract Stable V1 Extract Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/extract/css": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Extract structured data with CSS selectors (no LLM)",
+ "description": "Extract structured JSON from a URL using CSS selector schema.\n\nSchema format:\n{\n \"name\": \"products\",\n \"base_selector\": \".product-card\",\n \"fields\": [\n {\"name\": \"title\", \"selector\": \"h3\", \"type\": \"text\"},\n {\"name\": \"price\", \"selector\": \".price\", \"type\": \"text\", \"transform\": \"float\"},\n {\"name\": \"link\", \"selector\": \"a\", \"type\": \"attribute\", \"attribute\": \"href\"},\n {\"name\": \"in_stock\", \"selector\": \".stock\", \"type\": \"exists\"},\n ]\n}",
+ "operationId": "extract_css_v1_extract_css_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_extract_css_v1_extract_css_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Extract Css V1 Extract Css Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/extract/llm": {
+ "post": {
+ "tags": [
+ "Extraction",
+ "Extraction"
+ ],
+ "summary": "Extract with LLM + chunking strategies",
+ "description": "Extract structured data using LLM with intelligent chunking.\n\nChunks content by strategy (topic/sentence/regex), optionally filters\nby relevance to query, then extracts from each chunk.",
+ "operationId": "extract_llm_v1_extract_llm_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_extract_llm_v1_extract_llm_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Extract Llm V1 Extract Llm Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/freshness/check": {
+ "post": {
+ "tags": [
+ "Freshness",
+ "Freshness"
+ ],
+ "summary": "Check if content has changed since last scrape",
+ "description": "Check if content has changed since the last scrape.\n\nUses content fingerprinting (SHA256) to detect changes.\nIf no content is provided, does a quick HEAD check instead.",
+ "operationId": "freshness_check_v1_freshness_check_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_freshness_check_v1_freshness_check_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Freshness Check V1 Freshness Check Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/freshness/frequency": {
+ "post": {
+ "tags": [
+ "Freshness",
+ "Freshness"
+ ],
+ "summary": "Get adaptive scrape frequency recommendation",
+ "description": "Get an adaptive scrape frequency recommendation based on content volatility.\n\nVolatile pages (frequent changes) get shorter intervals.\nStable pages get longer intervals to save costs.",
+ "operationId": "freshness_frequency_v1_freshness_frequency_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_freshness_frequency_v1_freshness_frequency_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Freshness Frequency V1 Freshness Frequency Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/freshness/dashboard": {
+ "get": {
+ "tags": [
+ "Freshness",
+ "Freshness"
+ ],
+ "summary": "Get content staleness dashboard",
+ "description": "Get the content staleness dashboard.\n\nShows all tracked URLs, their last check time, age, and whether\nthey're stale (not checked in 24h).",
+ "operationId": "freshness_dashboard_v1_freshness_dashboard_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Freshness Dashboard V1 Freshness Dashboard Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/consent": {
+ "post": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Record user consent for data processing",
+ "description": "Record a user's consent for data processing (GDPR Art. 7).\n\nStores: user hash, purpose, timestamp, IP, user agent.\nConsent expires after 365 days.",
+ "operationId": "record_consent_endpoint_v1_gdpr_consent_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_record_consent_endpoint_v1_gdpr_consent_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Record Consent Endpoint V1 Gdpr Consent Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/consent/{user_id}": {
+ "get": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Check user consent status",
+ "description": "Check if a user has given valid consent for data processing.",
+ "operationId": "check_consent_endpoint_v1_gdpr_consent__user_id__get",
+ "parameters": [
+ {
+ "name": "user_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "User Id"
+ }
+ },
+ {
+ "name": "purpose",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "data_collection",
+ "title": "Purpose"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Check Consent Endpoint V1 Gdpr Consent User Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/consent/revoke": {
+ "post": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Revoke user consent",
+ "description": "Revoke a user's consent for a processing purpose (GDPR Art. 7(3)).",
+ "operationId": "revoke_consent_endpoint_v1_gdpr_consent_revoke_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_revoke_consent_endpoint_v1_gdpr_consent_revoke_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Revoke Consent Endpoint V1 Gdpr Consent Revoke Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/deletion/request": {
+ "post": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Request data deletion (right to erasure)",
+ "description": "Request deletion of all data associated with a user (GDPR Art. 17).\n\nCreates a deletion request that can be executed immediately or\nafter a holding period.",
+ "operationId": "request_deletion_endpoint_v1_gdpr_deletion_request_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_request_deletion_endpoint_v1_gdpr_deletion_request_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Request Deletion Endpoint V1 Gdpr Deletion Request Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/deletion/execute": {
+ "post": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Execute data deletion request",
+ "description": "Execute a pending deletion request, removing all user data.",
+ "operationId": "execute_deletion_endpoint_v1_gdpr_deletion_execute_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Request Id"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Execute Deletion Endpoint V1 Gdpr Deletion Execute Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/retention": {
+ "get": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Get data retention policy",
+ "description": "Get the current data retention policy.",
+ "operationId": "get_retention_policy_endpoint_v1_gdpr_retention_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Get Retention Policy Endpoint V1 Gdpr Retention Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/retention/apply": {
+ "post": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Apply retention policy to remove expired data",
+ "description": "Apply the retention policy, removing all expired data.",
+ "operationId": "apply_retention_v1_gdpr_retention_apply_post",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Apply Retention V1 Gdpr Retention Apply Post"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/gdpr/audit": {
+ "get": {
+ "tags": [
+ "GDPR",
+ "GDPR"
+ ],
+ "summary": "Get compliance audit log",
+ "description": "Get the compliance audit log for the specified period.",
+ "operationId": "get_audit_log_endpoint_v1_gdpr_audit_get",
+ "parameters": [
+ {
+ "name": "days_back",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 7,
+ "title": "Days Back"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Audit Log Endpoint V1 Gdpr Audit Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/destinations": {
+ "get": {
+ "tags": [
+ "Integrations",
+ "Integrations"
+ ],
+ "summary": "List supported data destinations",
+ "description": "List all supported data destinations and their config requirements.",
+ "operationId": "list_destinations_v1_destinations_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Destinations V1 Destinations Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/destination/send": {
+ "post": {
+ "tags": [
+ "Integrations",
+ "Integrations"
+ ],
+ "summary": "Send scraped data to a business destination",
+ "description": "Send extracted data to a business destination in one click.\n\nDestinations:\n- slack: Send to Slack channel via webhook\n- googlesheets: Write to Google Sheets (requires credentials)\n- airtable: Write to Airtable base (requires API key)\n- email: Send via email (requires SMTP config)\n\nConfig varies by destination:\n- slack: {\"webhook_url\": \"https://hooks.slack.com/...\"}\n- googlesheets: {\"spreadsheet_id\": \"...\", \"credentials_json\": \"...\"}\n- airtable: {\"base_id\": \"...\", \"table_name\": \"Table 1\", \"api_key\": \"...\"}\n- email: {\"recipient\": \"user@example.com\", \"subject\": \"Data Export\"}",
+ "operationId": "send_to_destination_v1_destination_send_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_send_to_destination_v1_destination_send_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Send To Destination V1 Destination Send Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/scrape-and-send": {
+ "post": {
+ "tags": [
+ "Integrations",
+ "Integrations"
+ ],
+ "summary": "Scrape a URL and send to a destination",
+ "description": "Scrape a URL and send the results to a business destination in one step.\n\nCombines /v1/scrape + /v1/destination/send into a single call.\nPerfect for non-technical users who just want data in their tools.",
+ "operationId": "scrape_and_send_v1_scrape_and_send_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_scrape_and_send_v1_scrape_and_send_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Scrape And Send V1 Scrape And Send Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/intel/snapshot": {
+ "post": {
+ "tags": [
+ "Intelligence",
+ "Intelligence"
+ ],
+ "summary": "Record a competitor data snapshot",
+ "description": "Record a data snapshot for a competitor.\n\nSnapshots are stored with timestamps and used for trend analysis,\nanomaly detection, and report generation.",
+ "operationId": "record_intel_snapshot_v1_intel_snapshot_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_record_intel_snapshot_v1_intel_snapshot_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Record Intel Snapshot V1 Intel Snapshot Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/intel/snapshots/{competitor_id}": {
+ "get": {
+ "tags": [
+ "Intelligence",
+ "Intelligence"
+ ],
+ "summary": "Get competitor snapshots",
+ "description": "Get historical snapshots for a competitor.",
+ "operationId": "get_intel_snapshots_v1_intel_snapshots__competitor_id__get",
+ "parameters": [
+ {
+ "name": "competitor_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Competitor Id"
+ }
+ },
+ {
+ "name": "limit",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 50,
+ "title": "Limit"
+ }
+ },
+ {
+ "name": "since_hours",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Since Hours"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Intel Snapshots V1 Intel Snapshots Competitor Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/intel/analyze": {
+ "post": {
+ "tags": [
+ "Intelligence",
+ "Intelligence"
+ ],
+ "summary": "Analyze competitor field for anomalies",
+ "description": "Analyze a specific field across a competitor's snapshots for anomalies.\n\nReturns statistics, z-score analysis, and anomaly detection.",
+ "operationId": "analyze_field_v1_intel_analyze_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_analyze_field_v1_intel_analyze_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Analyze Field V1 Intel Analyze Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/intel/report": {
+ "post": {
+ "tags": [
+ "Intelligence",
+ "Intelligence"
+ ],
+ "summary": "Generate a competitive intelligence report",
+ "description": "Generate a competitive intelligence report for tracked competitors.\n\nAnalyzes changes over the specified period and returns a structured report\nwith a natural-language summary.",
+ "operationId": "generate_report_v1_intel_report_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_generate_report_v1_intel_report_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Generate Report V1 Intel Report Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/job/{job_id}": {
+ "get": {
+ "tags": [
+ "Jobs",
+ "Jobs"
+ ],
+ "summary": "Get async job status and result",
+ "description": "Get async job status and result.",
+ "operationId": "get_job_v1_job__job_id__get",
+ "parameters": [
+ {
+ "name": "job_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Job Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Job V1 Job Job Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/jobs": {
+ "get": {
+ "tags": [
+ "Jobs",
+ "Jobs"
+ ],
+ "summary": "List jobs in a Pryfile",
+ "operationId": "list_jobs_v1_jobs_get",
+ "parameters": [
+ {
+ "name": "path",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "pry.yml",
+ "title": "Path"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response List Jobs V1 Jobs Get"
}
}
}
@@ -7631,6 +4523,7 @@
"/v1/actors/create": {
"post": {
"tags": [
+ "Marketplace",
"Marketplace"
],
"summary": "Create a new actor",
@@ -7675,6 +4568,7 @@
"/v1/actors": {
"get": {
"tags": [
+ "Marketplace",
"Marketplace"
],
"summary": "List actors in the marketplace",
@@ -7731,6 +4625,7 @@
"/v1/actors/{actor_id}/run": {
"post": {
"tags": [
+ "Marketplace",
"Marketplace"
],
"summary": "Run an actor",
@@ -7791,9 +4686,3103 @@
}
}
},
+ "/v1/monitor": {
+ "post": {
+ "tags": [
+ "Monitoring",
+ "Monitoring"
+ ],
+ "summary": "Create a scheduled content monitor",
+ "description": "Create a scheduled monitor that tracks content changes.\n\nArgs:\n name: Human-readable monitor name\n url: Target URL to monitor\n schedule_cron: Cron expression (default: every 6 hours)\n goal: Natural language goal for meaningful-change detection\n webhook: URL to notify on meaningful changes\n use_llm: Use LLM for change judging (slower but smarter)",
+ "operationId": "create_monitor_endpoint_v1_monitor_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_create_monitor_endpoint_v1_monitor_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Create Monitor Endpoint V1 Monitor Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/monitor/run": {
+ "post": {
+ "tags": [
+ "Monitoring",
+ "Monitoring"
+ ],
+ "summary": "Run a single monitor check",
+ "description": "Execute a monitor check immediately (outside of schedule).",
+ "operationId": "run_monitor_endpoint_v1_monitor_run_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Monitor Id"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Run Monitor Endpoint V1 Monitor Run Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/monitors": {
+ "get": {
+ "tags": [
+ "Monitoring",
+ "Monitoring"
+ ],
+ "summary": "List all monitors",
+ "description": "List all registered monitors.",
+ "operationId": "list_monitors_endpoint_v1_monitors_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Monitors Endpoint V1 Monitors Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/monitor/{monitor_id}": {
+ "delete": {
+ "tags": [
+ "Monitoring",
+ "Monitoring"
+ ],
+ "summary": "Delete a monitor",
+ "description": "Delete a monitor and its snapshots.",
+ "operationId": "delete_monitor_endpoint_v1_monitor__monitor_id__delete",
+ "parameters": [
+ {
+ "name": "monitor_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Monitor Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Delete Monitor Endpoint V1 Monitor Monitor Id Delete"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/monitor/check": {
+ "post": {
+ "tags": [
+ "Monitoring",
+ "Monitoring"
+ ],
+ "summary": "Run all due monitors",
+ "description": "Check all monitors and run any that are due based on their cron schedule.",
+ "operationId": "run_due_monitors_v1_monitor_check_post",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Run Due Monitors V1 Monitor Check Post"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/parse": {
+ "post": {
+ "tags": [
+ "Parsing",
+ "Parsing"
+ ],
+ "summary": "Parse a document (PDF, DOCX, image, CSV, JSON)",
+ "description": "Parse a document (PDF, DOCX, image, CSV, JSON) to text.",
+ "operationId": "parse_document_v1_parse_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ParseRequest"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Parse Document V1 Parse Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/markdown": {
+ "post": {
+ "tags": [
+ "Parsing",
+ "Parsing"
+ ],
+ "summary": "Generate markdown with content filtering strategies",
+ "description": "Generate markdown with configurable content filtering.\n\nModes:\n- raw: Unfiltered markdown\n- fit: Prune boilerplate (nav, ads, footers)\n- bm25: Filter by BM25 relevance to query (requires query param)",
+ "operationId": "generate_markdown_v1_markdown_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_generate_markdown_v1_markdown_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Generate Markdown V1 Markdown Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/shadow-dom": {
+ "post": {
+ "tags": [
+ "Parsing",
+ "Parsing"
+ ],
+ "summary": "Extract content from Shadow DOM",
+ "description": "Scrape a page and extract content from Shadow DOM components.\n\nUseful for modern web apps built with Lit, web components, or\nframeworks that use Shadow DOM encapsulation.",
+ "operationId": "extract_shadow_dom_v1_shadow_dom_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_extract_shadow_dom_v1_shadow_dom_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Extract Shadow Dom V1 Shadow Dom Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipeline/hook": {
+ "post": {
+ "tags": [
+ "Pipeline",
+ "Pipeline"
+ ],
+ "summary": "Register a hook function",
+ "description": "Register a JavaScript hook function at a pipeline hook point.\n\nHook points: before_scrape, after_response, before_parse, after_parse,\n before_extract, after_extract, before_return, on_error",
+ "operationId": "register_hook_v1_pipeline_hook_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_register_hook_v1_pipeline_hook_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Register Hook V1 Pipeline Hook Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipeline/hooks": {
+ "get": {
+ "tags": [
+ "Pipeline",
+ "Pipeline"
+ ],
+ "summary": "List all registered hooks",
+ "description": "List all registered hooks in the pipeline.",
+ "operationId": "list_pipeline_hooks_v1_pipeline_hooks_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Pipeline Hooks V1 Pipeline Hooks Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipeline/run": {
+ "post": {
+ "tags": [
+ "Pipeline",
+ "Pipeline"
+ ],
+ "summary": "Run pipeline hooks for testing",
+ "description": "Run hooks at a given pipeline point for testing.",
+ "operationId": "run_pipeline_test_v1_pipeline_run_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_run_pipeline_test_v1_pipeline_run_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Run Pipeline Test V1 Pipeline Run Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipelines/steps": {
+ "get": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "List all available pipeline step types",
+ "description": "List all available step types for the visual pipeline builder.\n\nEach step type includes: name, icon, description, required inputs with types,\nand expected outputs. A UI renders these as drag-and-drop blocks.",
+ "operationId": "list_step_types_v1_pipelines_steps_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Step Types V1 Pipelines Steps Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipelines/validate": {
+ "post": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "Validate a pipeline definition",
+ "description": "Validate a pipeline definition for correctness.",
+ "operationId": "validate_pipeline_endpoint_v1_pipelines_validate_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Pipeline"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Validate Pipeline Endpoint V1 Pipelines Validate Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipelines/run": {
+ "post": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "Execute a pipeline",
+ "description": "Execute a pipeline definition.\n\nSteps run sequentially. Each step's output is available as\n{{step_id.output_key}} in subsequent step templates.\n\nExample pipeline:\n{\n \"name\": \"Monitor Competitor Pricing\",\n \"steps\": [\n {\"id\": \"scrape_amazon\", \"type\": \"scrape\", \"inputs\": {\"url\": \"https://amazon.com/product\"}},\n {\"id\": \"extract\", \"type\": \"extract_css\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"schema\": {...}}},\n {\"id\": \"quality\", \"type\": \"quality_check\", \"inputs\": {\"url\": \"{{scrape_amazon.url}}\", \"data\": \"{{extract.items}}\"}},\n {\"id\": \"notify\", \"type\": \"send_slack\", \"inputs\": {\"webhook_url\": \"https://hooks.slack.com/...\", \"message\": \"{{quality.quality_score}}\"}},\n ]\n}",
+ "operationId": "run_pipeline_endpoint_v1_pipelines_run_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_run_pipeline_endpoint_v1_pipelines_run_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Run Pipeline Endpoint V1 Pipelines Run Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipelines/save": {
+ "post": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "Save a pipeline definition",
+ "description": "Save a pipeline definition for later use.",
+ "operationId": "save_pipeline_endpoint_v1_pipelines_save_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Pipeline"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Save Pipeline Endpoint V1 Pipelines Save Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipelines": {
+ "get": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "List saved pipelines",
+ "description": "List all saved pipeline definitions.",
+ "operationId": "list_pipelines_endpoint_v1_pipelines_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Pipelines Endpoint V1 Pipelines Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipelines/{pipeline_id}": {
+ "get": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "Get a saved pipeline",
+ "description": "Get a saved pipeline definition by ID.",
+ "operationId": "get_pipeline_endpoint_v1_pipelines__pipeline_id__get",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Pipeline Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Pipeline Endpoint V1 Pipelines Pipeline Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "delete": {
+ "tags": [
+ "Pipelines",
+ "Pipelines"
+ ],
+ "summary": "Delete a saved pipeline",
+ "description": "Delete a saved pipeline definition.",
+ "operationId": "delete_pipeline_endpoint_v1_pipelines__pipeline_id__delete",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Pipeline Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Delete Pipeline Endpoint V1 Pipelines Pipeline Id Delete"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/providers": {
+ "get": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "List all available proxy providers",
+ "description": "List free + premium proxy providers with affiliate details.",
+ "operationId": "list_proxy_providers_v1_proxy_providers_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Proxy Providers V1 Proxy Providers Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/signup": {
+ "post": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Open affiliate signup link for a provider",
+ "description": "Get the affiliate signup URL for a proxy provider.\n\nRecords a click for revenue tracking. The user can then sign up at that URL\nand come back to configure credentials.",
+ "operationId": "proxy_signup_v1_proxy_signup_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Provider"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Proxy Signup V1 Proxy Signup Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/referrals": {
+ "get": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "List proxy provider affiliate referrals",
+ "description": "Return the curated proxy provider affiliate catalog (proxy_referrals.py).\n\nThis is the marketing-friendly subset: commission, promo codes, tier\n(premium/standard/budget), and referral URLs. Useful for a \"Sign up via Pry\nand we get a cut\" UI, or for showing users premium options when their free\nproxy fails.\n\nThe full connection metadata (host, port, auth) lives in /v1/proxy/providers.",
+ "operationId": "list_proxy_referrals_v1_proxy_referrals_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Proxy Referrals V1 Proxy Referrals Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/referrals/{tag}": {
+ "get": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Get a single proxy provider referral",
+ "description": "Return the affiliate info for a single proxy provider by tag.",
+ "operationId": "get_proxy_referral_v1_proxy_referrals__tag__get",
+ "parameters": [
+ {
+ "name": "tag",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Tag"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Proxy Referral V1 Proxy Referrals Tag Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/configure": {
+ "post": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Configure proxy credentials",
+ "description": "Configure credentials for a premium proxy provider.\n\nAfter signing up via /v1/proxy/signup, the user provides their credentials here.",
+ "operationId": "proxy_configure_v1_proxy_configure_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_proxy_configure_v1_proxy_configure_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Proxy Configure V1 Proxy Configure Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/test": {
+ "get": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Test the active proxy",
+ "description": "Test the currently configured proxy and return its public IP.",
+ "operationId": "proxy_test_v1_proxy_test_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Proxy Test V1 Proxy Test Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/status": {
+ "get": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Get current proxy status",
+ "description": "Get current proxy configuration and available providers.",
+ "operationId": "proxy_status_v1_proxy_status_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Proxy Status V1 Proxy Status Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/recommend": {
+ "post": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Get proxy recommendation after a block",
+ "description": "After a scrape fails with anti-bot detection, get a recommendation\nfor which premium proxy provider to sign up with.",
+ "operationId": "proxy_recommend_v1_proxy_recommend_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Last Error",
+ "default": ""
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Proxy Recommend V1 Proxy Recommend Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/proxy/clicks": {
+ "get": {
+ "tags": [
+ "Proxy",
+ "Proxy"
+ ],
+ "summary": "Get recent proxy referral clicks",
+ "description": "Get recent proxy referral clicks for revenue tracking.",
+ "operationId": "proxy_clicks_v1_proxy_clicks_get",
+ "parameters": [
+ {
+ "name": "days_back",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 30,
+ "title": "Days Back"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Proxy Clicks V1 Proxy Clicks Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/quality/check": {
+ "post": {
+ "tags": [
+ "Quality",
+ "Quality"
+ ],
+ "summary": "Run data quality check on extraction results",
+ "description": "Run a full data quality check on extraction results.\n\nMetrics:\n- Completeness: what % of expected fields have values\n- Schema adherence: field types match expectations\n- Freshness: how old is the data\n- Anomaly detection: what changed since last extraction\n\nUse this to validate data BEFORE sending to downstream systems.",
+ "operationId": "quality_check_v1_quality_check_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_quality_check_v1_quality_check_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Quality Check V1 Quality Check Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/quality/stats": {
+ "get": {
+ "tags": [
+ "Quality",
+ "Quality"
+ ],
+ "summary": "Get quality statistics for all checked URLs",
+ "description": "Get aggregate quality statistics across all checked URLs.",
+ "operationId": "quality_stats_v1_quality_stats_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Quality Stats V1 Quality Stats Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/reconcile": {
+ "post": {
+ "tags": [
+ "Reconciliation",
+ "Reconciliation"
+ ],
+ "summary": "Reconcile records from multiple sources into unified entities",
+ "description": "Reconcile records from multiple sources into matched entities.\n\nMatches records across sources using identity field similarity,\nnormalizes to a unified vertical schema, and returns entity groups\nwith confidence scores.\n\nVerticals: product, job, real_estate, review",
+ "operationId": "reconcile_endpoint_v1_reconcile_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_reconcile_endpoint_v1_reconcile_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Reconcile Endpoint V1 Reconcile Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/reconcile/schemas": {
+ "get": {
+ "tags": [
+ "Reconciliation",
+ "Reconciliation"
+ ],
+ "summary": "List supported reconciliation schemas",
+ "description": "List all supported vertical schemas for entity reconciliation.",
+ "operationId": "list_schemas_v1_reconcile_schemas_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Schemas V1 Reconcile Schemas Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/record/start": {
+ "post": {
+ "tags": [
+ "Recorder",
+ "Recorder"
+ ],
+ "summary": "Start recording browser actions",
+ "operationId": "start_recording_v1_record_start_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Session Id"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Start Recording V1 Record Start Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/record/step": {
+ "post": {
+ "tags": [
+ "Recorder",
+ "Recorder"
+ ],
+ "summary": "Record a browser action step",
+ "operationId": "record_step_v1_record_step_post",
+ "parameters": [
+ {
+ "name": "selector",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "",
+ "title": "Selector"
+ }
+ },
+ {
+ "name": "value",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "",
+ "title": "Value"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_record_step_v1_record_step_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Record Step V1 Record Step Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/record/export": {
+ "post": {
+ "tags": [
+ "Recorder",
+ "Recorder"
+ ],
+ "summary": "Export recording as script",
+ "operationId": "export_recording_v1_record_export_post",
+ "parameters": [
+ {
+ "name": "fmt",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "json",
+ "title": "Fmt"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Session Id"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Export Recording V1 Record Export Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/record/clear": {
+ "post": {
+ "tags": [
+ "Recorder",
+ "Recorder"
+ ],
+ "summary": "Clear recorded actions",
+ "operationId": "clear_recording_v1_record_clear_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Session Id"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Clear Recording V1 Record Clear Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/referrals/catalog": {
+ "get": {
+ "tags": [
+ "Referrals",
+ "Referrals"
+ ],
+ "summary": "Get all available referral/affiliate programs",
+ "description": "List all referral programs Pry supports.\n\n60+ providers across categories: LLM, hosting, domains, CDN, email,\nmonitoring, proxies, voice, media, devtools, search, CAPTCHA.",
+ "operationId": "get_referral_catalog_v1_referrals_catalog_get",
+ "parameters": [
+ {
+ "name": "category",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "",
+ "title": "Category"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Referral Catalog V1 Referrals Catalog Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/referrals/stats": {
+ "get": {
+ "tags": [
+ "Referrals",
+ "Referrals"
+ ],
+ "summary": "Get referral click and conversion stats",
+ "description": "Get referral tracking statistics for the last N days.",
+ "operationId": "get_referral_stats_v1_referrals_stats_get",
+ "parameters": [
+ {
+ "name": "days_back",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "integer",
+ "default": 30,
+ "title": "Days Back"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Referral Stats V1 Referrals Stats Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/referrals/click": {
+ "post": {
+ "tags": [
+ "Referrals",
+ "Referrals"
+ ],
+ "summary": "Record a referral link click",
+ "description": "Record when a user clicks a referral link. Returns tracking ID.",
+ "operationId": "record_referral_click_v1_referrals_click_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_record_referral_click_v1_referrals_click_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Record Referral Click V1 Referrals Click Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/referrals/convert": {
+ "post": {
+ "tags": [
+ "Referrals",
+ "Referrals"
+ ],
+ "summary": "Record a referral conversion",
+ "description": "Record that a referral click resulted in a conversion.",
+ "operationId": "record_referral_conversion_v1_referrals_convert_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_record_referral_conversion_v1_referrals_convert_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Record Referral Conversion V1 Referrals Convert Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/reports/generate": {
+ "post": {
+ "tags": [
+ "Reports",
+ "Reports"
+ ],
+ "summary": "Generate a white-label report from scraped data",
+ "description": "Generate a white-label report from scraped data.\n\nReport types:\n- competitive_analysis: Competitor pricing and activity overview\n- price_monitor: Product price change tracking with visual indicators\n- seo_audit: SEO element analysis with change detection\n- content_tracker: Content change monitoring across pages\n\nBranding (optional): {\"agency_name\": \"...\", \"brand_color\": \"#hex\", \"logo_url\": \"...\"}",
+ "operationId": "generate_report_endpoint_v1_reports_generate_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_generate_report_endpoint_v1_reports_generate_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Generate Report Endpoint V1 Reports Generate Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/reports": {
+ "get": {
+ "tags": [
+ "Reports",
+ "Reports"
+ ],
+ "summary": "List generated reports",
+ "description": "List all previously generated reports.",
+ "operationId": "list_reports_endpoint_v1_reports_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Reports Endpoint V1 Reports Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/report/{report_id}": {
+ "get": {
+ "tags": [
+ "Reports",
+ "Reports"
+ ],
+ "summary": "Get a generated report",
+ "description": "Get the HTML content of a generated report.",
+ "operationId": "get_report_v1_report__report_id__get",
+ "parameters": [
+ {
+ "name": "report_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Report Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "title": "Response Get Report V1 Report Report Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/review/submit": {
+ "post": {
+ "tags": [
+ "Review",
+ "Review"
+ ],
+ "summary": "Submit extracted data for human review",
+ "description": "Submit extracted data for human review before delivery.\n\nUse this when extraction confidence is low or anomalies were detected.\nData will be held in the review queue until approved or rejected.",
+ "operationId": "review_submit_v1_review_submit_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_review_submit_v1_review_submit_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Review Submit V1 Review Submit Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/review/{review_id}/approve": {
+ "post": {
+ "tags": [
+ "Review",
+ "Review"
+ ],
+ "summary": "Approve a review item",
+ "description": "Approve a review item, allowing data to proceed to delivery.",
+ "operationId": "review_approve_v1_review__review_id__approve_post",
+ "parameters": [
+ {
+ "name": "review_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Review Id"
+ }
+ }
+ ],
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_review_approve_v1_review__review_id__approve_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Review Approve V1 Review Review Id Approve Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/review/{review_id}/reject": {
+ "post": {
+ "tags": [
+ "Review",
+ "Review"
+ ],
+ "summary": "Reject a review item",
+ "description": "Reject a review item, blocking data delivery.",
+ "operationId": "review_reject_v1_review__review_id__reject_post",
+ "parameters": [
+ {
+ "name": "review_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Review Id"
+ }
+ }
+ ],
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_review_reject_v1_review__review_id__reject_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Review Reject V1 Review Review Id Reject Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/reviews": {
+ "get": {
+ "tags": [
+ "Review",
+ "Review"
+ ],
+ "summary": "List reviews in the queue",
+ "description": "List reviews, optionally filtered by status (pending/approved/rejected).",
+ "operationId": "list_reviews_v1_reviews_get",
+ "parameters": [
+ {
+ "name": "status",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Status"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response List Reviews V1 Reviews Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/review/{review_id}": {
+ "get": {
+ "tags": [
+ "Review",
+ "Review"
+ ],
+ "summary": "Get review details",
+ "description": "Get full details of a review item including the data payload.",
+ "operationId": "get_review_v1_review__review_id__get",
+ "parameters": [
+ {
+ "name": "review_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Review Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Get Review V1 Review Review Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/extract-with-review": {
+ "post": {
+ "tags": [
+ "Review",
+ "Review"
+ ],
+ "summary": "Extract with automatic human review routing",
+ "description": "Extract data with automatic quality check and human review routing.\n\nHigh-confidence results are auto-approved.\nLow-confidence results are auto-rejected.\nMedium-confidence results go to the human review queue with Slack notification.",
+ "operationId": "extract_with_review_v1_extract_with_review_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_extract_with_review_v1_extract_with_review_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Extract With Review V1 Extract With Review Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/seo/analyze": {
+ "post": {
+ "tags": [
+ "SEO",
+ "SEO"
+ ],
+ "summary": "Analyze SEO elements from a URL",
+ "description": "Analyze all SEO elements from a URL.\n\nReturns: title, meta description, keywords, headings (H1/H2),\ncanonical, OG tags, Twitter cards, word count, link counts,\nschema markup, hreflang tags.",
+ "operationId": "seo_analyze_v1_seo_analyze_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Seo Analyze V1 Seo Analyze Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/seo/track": {
+ "post": {
+ "tags": [
+ "SEO",
+ "SEO"
+ ],
+ "summary": "Track SEO changes since last scan",
+ "description": "Track SEO changes since the last scan of this URL.\n\nCompares current SEO elements to previous snapshot and reports\nwhat changed (title, description, headings, etc.).",
+ "operationId": "seo_track_v1_seo_track_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Seo Track V1 Seo Track Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/seo/keywords": {
+ "post": {
+ "tags": [
+ "SEO",
+ "SEO"
+ ],
+ "summary": "Analyze keyword presence in URL content",
+ "description": "Analyze which keywords a URL targets.\n\nChecks each keyword for:\n- Presence in title tag\n- Presence in H1 headings\n- Presence in meta description\n- Frequency in body content\n- Keyword density percentage",
+ "operationId": "seo_keywords_v1_seo_keywords_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_seo_keywords_v1_seo_keywords_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Seo Keywords V1 Seo Keywords Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/ultimate-scrape": {
+ "post": {
+ "tags": [
+ "Scraping",
+ "Scraping"
+ ],
+ "summary": "Scrape with 10-tier anti-bot fallback system",
+ "description": "Scrape any URL using Pry's ultimate 10-tier anti-detection system.\n\nAutomatically tries: direct \u2192 cloudscraper \u2192 FlareSolverr \u2192\nundetected-chromedriver \u2192 Playwright \u2192 Googlebot \u2192 Archive.org \u2192 Google Cache\n\nReturns the first successful result with the method used.",
+ "operationId": "ultimate_scrape_v1_ultimate_scrape_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_ultimate_scrape_v1_ultimate_scrape_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Ultimate Scrape V1 Ultimate Scrape Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/capture/network": {
+ "post": {
+ "tags": [
+ "Scraping",
+ "Scraping"
+ ],
+ "summary": "Extract API calls and network patterns from a page",
+ "description": "Extract API calls, GraphQL queries, and network patterns from a page.\n\nUseful for understanding how SPAs load data and finding hidden API endpoints.",
+ "operationId": "capture_network_v1_capture_network_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Capture Network V1 Capture Network Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/crawl/adaptive": {
+ "post": {
+ "tags": [
+ "Scraping",
+ "Scraping"
+ ],
+ "summary": "Crawl with adaptive stopping based on content relevance",
+ "description": "Crawl a website with adaptive stopping.\n\nUses information foraging theory to decide when to stop:\n- Stops when content relevance drops below threshold\n- Stops when information gain diminishes\n- Respects max_pages and max_depth limits\n- Ideal for targeted data collection (pricing, docs, products)",
+ "operationId": "adaptive_crawl_v1_crawl_adaptive_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_adaptive_crawl_v1_crawl_adaptive_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Adaptive Crawl V1 Crawl Adaptive Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/session/create": {
+ "post": {
+ "tags": [
+ "Sessions",
+ "Sessions"
+ ],
+ "summary": "Create a persistent browser session",
+ "description": "Create a persistent browser session.",
+ "operationId": "create_session_v1_session_create_post",
+ "parameters": [
+ {
+ "name": "persist",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "boolean",
+ "default": true,
+ "title": "Persist"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Url"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Create Session V1 Session Create Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/session/destroy": {
+ "post": {
+ "tags": [
+ "Sessions",
+ "Sessions"
+ ],
+ "summary": "Destroy a browser session with optional save",
+ "description": "Destroy a browser session. Optionally save state first.",
+ "operationId": "destroy_session_v1_session_destroy_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_destroy_session_v1_session_destroy_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Destroy Session V1 Session Destroy Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/sessions": {
+ "get": {
+ "tags": [
+ "Sessions",
+ "Sessions"
+ ],
+ "summary": "List all persistent sessions",
+ "description": "List all persistent sessions (active + saved).",
+ "operationId": "list_sessions_v1_sessions_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response List Sessions V1 Sessions Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/session/save": {
+ "post": {
+ "tags": [
+ "Sessions",
+ "Sessions"
+ ],
+ "summary": "Save current session state to disk",
+ "description": "Save a session's current state (cookies, storage) to disk.",
+ "operationId": "save_session_state_v1_session_save_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Session Id"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Save Session State V1 Session Save Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/session/restore": {
+ "post": {
+ "tags": [
+ "Sessions",
+ "Sessions"
+ ],
+ "summary": "Restore a saved session",
+ "description": "Restore a session from disk into a browser context.",
+ "operationId": "restore_session_v1_session_restore_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Session Id"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Restore Session V1 Session Restore Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/share": {
+ "post": {
+ "tags": [
+ "Share",
+ "Share"
+ ],
+ "summary": "Share scraped content via link",
+ "operationId": "share_scrape_v1_share_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Data"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Share Scrape V1 Share Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/share/{share_id}": {
+ "get": {
+ "tags": [
+ "Share",
+ "Share"
+ ],
+ "summary": "View shared content",
+ "operationId": "view_share_share__share_id__get",
+ "parameters": [
+ {
+ "name": "share_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Share Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {}
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v0/stats": {
+ "get": {
+ "tags": [
+ "Stats",
+ "Stats"
+ ],
+ "summary": "Get cache, rate limiter, and session stats",
+ "operationId": "stats_v0_stats_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Stats V0 Stats Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/structure/check": {
+ "post": {
+ "tags": [
+ "Structure",
+ "Structure"
+ ],
+ "summary": "Check if CSS selectors still match on a page",
+ "description": "Check if CSS selectors still match on a page.\n\nUse this to verify your scraper templates still work after\na site redesign. Returns per-selector match status.\n\nSelectors format: [{\"name\": \"title\", \"selector\": \"h1\", \"type\": \"css\"}]",
+ "operationId": "structure_check_v1_structure_check_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_structure_check_v1_structure_check_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Structure Check V1 Structure Check Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/structure/monitor": {
+ "post": {
+ "tags": [
+ "Structure",
+ "Structure"
+ ],
+ "summary": "Monitor page structure changes over time",
+ "description": "Monitor page structure changes and detect broken selectors.\n\nCompares current selector match status to previous checks.\nAlerts when selectors stop matching (site redesign detected).\n\nUse this as a pre-scrape health check for your templates.",
+ "operationId": "structure_monitor_v1_structure_monitor_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_structure_monitor_v1_structure_monitor_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Structure Monitor V1 Structure Monitor Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/structure/check-template": {
+ "post": {
+ "tags": [
+ "Structure",
+ "Structure"
+ ],
+ "summary": "Verify a scraper template still works",
+ "description": "Check if a pre-built scraper template still works against a URL.\n\nExtracts the template's selectors and checks each one.\nIf selectors fail, the template needs updating.",
+ "operationId": "structure_check_template_v1_structure_check_template_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_structure_check_template_v1_structure_check_template_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Structure Check Template V1 Structure Check Template Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/training/classify-license": {
+ "post": {
+ "tags": [
+ "Training",
+ "Training"
+ ],
+ "summary": "Classify content license for AI training",
+ "description": "Classify the license of scraped content for AI training compliance.\n\nReturns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use),\ntier (permissive/copyleft/restrictive/conditional), and confidence.",
+ "operationId": "classify_license_endpoint_v1_training_classify_license_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "string",
+ "title": "Text"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Classify License Endpoint V1 Training Classify License Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/training/clean": {
+ "post": {
+ "tags": [
+ "Training",
+ "Training"
+ ],
+ "summary": "Strip PII and copyright from content",
+ "description": "Strip PII and copyright content for AI training clean room.\n\nRemoves emails, phones, SSNs, credit cards, IPs, and (optionally) names.\nAlso strips copyright notices and near-verbatim copyright content.",
+ "operationId": "clean_content_v1_training_clean_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_clean_content_v1_training_clean_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Clean Content V1 Training Clean Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/training/export": {
+ "post": {
+ "tags": [
+ "Training",
+ "Training"
+ ],
+ "summary": "Export a clean AI training dataset",
+ "description": "Export scraped content as a clean AI training dataset.\n\nEach record should have: content, url, and optional metadata.\n\nFeatures:\n- Per-record provenance tracking (source URL, timestamp, extraction method)\n- PII stripping (email, phone, SSN, CC, IP)\n- Copyright verbatim text removal\n- License classification\n- Compliance report generation",
+ "operationId": "export_dataset_v1_training_export_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_export_dataset_v1_training_export_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Export Dataset V1 Training Export Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/training/compliance/{dataset_id}": {
+ "get": {
+ "tags": [
+ "Training",
+ "Training"
+ ],
+ "summary": "Generate compliance report for a dataset",
+ "description": "Generate a compliance report for an exported training dataset.\n\nReport includes: data provenance, PII/copyright removal stats,\nlicense classification, and legal recommendations.",
+ "operationId": "compliance_report_v1_training_compliance__dataset_id__get",
+ "parameters": [
+ {
+ "name": "dataset_id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string",
+ "title": "Dataset Id"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Compliance Report V1 Training Compliance Dataset Id Get"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/transform": {
+ "post": {
+ "tags": [
+ "Transform",
+ "Transform"
+ ],
+ "summary": "Transform data to multiple formats",
+ "operationId": "transform_data_v1_transform_post",
+ "parameters": [
+ {
+ "name": "format",
+ "in": "query",
+ "required": false,
+ "schema": {
+ "type": "string",
+ "default": "csv",
+ "title": "Format"
+ }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "additionalProperties": true
+ },
+ "title": "Data"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "additionalProperties": true,
+ "title": "Response Transform Data V1 Transform Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/pipe": {
+ "post": {
+ "tags": [
+ "Transform",
+ "Transform"
+ ],
+ "summary": "Scrape and transform via data pipeline",
+ "operationId": "data_pipeline_v1_pipe_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Data"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Data Pipeline V1 Pipe Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/vision": {
+ "post": {
+ "tags": [
+ "Vision",
+ "Vision"
+ ],
+ "summary": "Analyze an image with a free vision model",
+ "description": "Analyze an image with a free vision model.\n\nProvide ONE of: image (base64 or data URI), url (auto-screenshot),\nor file_path (local PNG/JPG).\n\nAuto-falls-back across 5 free OpenRouter vision models if the\nrequested one is rate-limited.",
+ "operationId": "vision_v1_vision_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_vision_v1_vision_post"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Vision V1 Vision Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/vision/models": {
+ "get": {
+ "tags": [
+ "Vision",
+ "Vision"
+ ],
+ "summary": "List free vision-capable models on OpenRouter",
+ "description": "List free vision-capable models on OpenRouter.",
+ "operationId": "vision_models_v1_vision_models_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response Vision Models V1 Vision Models Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/v1/webhooks/test": {
"post": {
"tags": [
+ "Webhooks",
"Webhooks"
],
"summary": "Test webhook delivery",
@@ -7838,6 +7827,7 @@
"/v1/webhooks/dead-letter": {
"get": {
"tags": [
+ "Webhooks",
"Webhooks"
],
"summary": "Get failed webhook deliveries",
@@ -7858,6 +7848,305 @@
}
}
}
+ },
+ "/v1/x402/pricing": {
+ "get": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Get x402 pricing for all paid operations",
+ "description": "Get the price list for pay-per-scrape operations.",
+ "operationId": "x402_pricing_v1_x402_pricing_get",
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Pricing V1 X402 Pricing Get"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/x402/payment": {
+ "post": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Create a x402 payment request",
+ "description": "Create a x402 payment request for a paid operation.\n\nReturns payment details (wallet, amount, asset) for the client to pay.",
+ "operationId": "x402_payment_request_v1_x402_payment_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_x402_payment_request_v1_x402_payment_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Payment Request V1 X402 Payment Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/x402/verify": {
+ "post": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Verify a x402 payment was settled",
+ "description": "Verify a x402 payment has been settled on-chain via facilitator router.",
+ "operationId": "x402_verify_payment_v1_x402_verify_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_x402_verify_payment_v1_x402_verify_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Verify Payment V1 X402 Verify Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/x402/require-payment": {
+ "post": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Generate a 402 Payment Required response",
+ "description": "Generate a 402 Payment Required response for a paid endpoint.",
+ "operationId": "x402_require_payment_v1_x402_require_payment_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_x402_require_payment_v1_x402_require_payment_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Require Payment V1 X402 Require Payment Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/x402/batch-payment": {
+ "post": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Create a batch x402 payment",
+ "description": "Create a single x402 payment covering multiple operations.\n\nReturns a PaymentRequired body with the combined amount. After paying,\nsubmit the tx to POST /v1/x402/batch-verify.",
+ "operationId": "x402_batch_payment_v1_x402_batch_payment_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Payload"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Batch Payment V1 X402 Batch Payment Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/x402/batch-verify": {
+ "post": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Verify a batch x402 payment",
+ "description": "Verify the on-chain payment for a batch and mark it paid.",
+ "operationId": "x402_batch_verify_v1_x402_batch_verify_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Payload"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Batch Verify V1 X402 Batch Verify Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "/v1/x402/pay": {
+ "post": {
+ "tags": [
+ "x402",
+ "x402"
+ ],
+ "summary": "Process x402 payment and get access token",
+ "description": "Process an x402 payment and return an access token.\n\nFlow:\n1. User gets 402 from a paid endpoint\n2. User sends USDC to the wallet in the 402 response\n3. User calls this endpoint with the tx_hash\n4. Pry verifies the transaction through the facilitator router\n5. Returns access token (payment_id) to use in X-Payment-Hash header",
+ "operationId": "x402_pay_v1_x402_pay_post",
+ "requestBody": {
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/Body_x402_pay_v1_x402_pay_post"
+ }
+ }
+ },
+ "required": true
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "additionalProperties": true,
+ "type": "object",
+ "title": "Response X402 Pay V1 X402 Pay Post"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/HTTPValidationError"
+ }
+ }
+ }
+ }
+ }
+ }
}
},
"components": {
@@ -8030,6 +8319,37 @@
],
"title": "BatchFileRequest"
},
+ "BatchItem": {
+ "properties": {
+ "template_id": {
+ "type": "string",
+ "title": "Template Id"
+ },
+ "url": {
+ "type": "string",
+ "title": "Url"
+ },
+ "options": {
+ "anyOf": [
+ {
+ "additionalProperties": true,
+ "type": "object"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Options"
+ }
+ },
+ "type": "object",
+ "required": [
+ "template_id",
+ "url"
+ ],
+ "title": "BatchItem",
+ "description": "A single template execution request within a batch."
+ },
"Body_adaptive_crawl_v1_crawl_adaptive_post": {
"properties": {
"url": {
@@ -10356,4 +10676,4 @@
}
}
}
-}
+}
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index ccc695c..efab7e6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,6 +50,9 @@ dependencies = [
"structlog>=24.0.0",
"sqlalchemy>=2.0.0",
"aiosqlite>=0.19.0",
+ "prometheus-client>=0.21.0",
+ "opentelemetry-api>=1.29.0",
+ "opentelemetry-sdk>=1.29.0",
]
[project.optional-dependencies]
diff --git a/requirements.lock b/requirements.lock
new file mode 100644
index 0000000..3805281
--- /dev/null
+++ b/requirements.lock
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+# Part of Pry โ https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Generated from pyproject.toml; pinned to currently installed versions.
+
+fastapi==0.128.8
+uvicorn==0.49.0
+trafilatura==2.1.0
+readability-lxml==0.8.4.1
+lxml==6.1.1
+httpx==0.28.1
+markdownify==1.2.3
+pydantic==2.12.5
+pydantic-settings==2.14.2
+playwright==1.60.0
+redis==8.0.0
+pypdf==6.14.2
+python-docx==1.2.0
+tiktoken==0.12.0
+pillow==12.2.0
+click==8.1.8
+pyyaml==6.0.3
+pandas==3.0.3
+anyio==4.14.1
+croniter==6.2.3
+structlog==26.1.0
+sqlalchemy==2.0.50
+aiosqlite==0.22.1
+prometheus-client==0.25.0
+opentelemetry-api==1.37.0
+opentelemetry-sdk==1.37.0
diff --git a/requirements.txt b/requirements.txt
index 966ffe3..ad5d64e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -27,3 +27,6 @@ structlog>=24.0.0
sqlalchemy>=2.0.0
aiosqlite>=0.19.0
alembic>=1.14.0
+prometheus-client>=0.21.0
+opentelemetry-api>=1.29.0
+opentelemetry-sdk>=1.29.0
diff --git a/routers/__init__.py b/routers/__init__.py
index c8f45ce..79ca4fd 100644
--- a/routers/__init__.py
+++ b/routers/__init__.py
@@ -43,7 +43,56 @@ Licensed under MIT. See LICENSE.
# Licensed under MIT. See LICENSE.
__all__: list[str] = [
- "auth_router",
- "health_router",
- "templates_router",
+ "advanced_router",
+ "agency_router",
+ "ai_router",
+ "alerts_router",
+ "analysis_router",
+ "auth_api_router",
+ "automation_router",
+ "batch_router",
+ "circuit_breaker_router",
+ "commerce_router",
+ "compliance_router",
+ "config_router",
+ "costing_router",
+ "crm_router",
+ "dashboard_router",
+ "email_router",
+ "enrichment_router",
+ "execute_router",
+ "export_router",
+ "extraction_router",
+ "freshness_router",
+ "gdpr_router",
+ "health_api_router",
+ "integrations_router",
+ "intelligence_router",
+ "jobs_router",
+ "marketplace_router",
+ "monitoring_router",
+ "parsing_router",
+ "pipeline_router",
+ "pipelines_router",
+ "proxy_router",
+ "quality_router",
+ "reconciliation_router",
+ "recorder_router",
+ "referrals_router",
+ "reports_router",
+ "review_router",
+ "scraping_api_router",
+ "seo_router",
+ "sessions_router",
+ "share_router",
+ "stats_router",
+ "structure_router",
+ "system_router",
+ "templates_api_router",
+ "training_router",
+ "transform_router",
+ "untagged_router",
+ "vision_router",
+ "webhooks_router",
+ "x402_router",
]
diff --git a/routers/advanced.py b/routers/advanced.py
new file mode 100644
index 0000000..76d2675
--- /dev/null
+++ b/routers/advanced.py
@@ -0,0 +1,329 @@
+"""Pry โ Advanced router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import base64
+import logging
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Body
+
+from client import get_client
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Advanced"])
+
+@router.post(
+ "/v1/tls/impersonate",
+ tags=["Advanced"],
+ summary="Fetch a URL with TLS fingerprint impersonation",
+)
+async def tls_impersonate(
+ url: str = Body(...),
+ impersonate: str = Body("chrome131"),
+ proxy: str = Body(""),
+) -> dict[str, Any]:
+ """Fetch a URL while impersonating a real browser's TLS fingerprint.
+ Bypasses JA3/JA4 fingerprinting that blocks 80%+ of bot traffic."""
+ from tls_fingerprint import TLSScraper
+
+ s = TLSScraper()
+ if not s.is_available():
+ return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"}
+ result = await s.fetch(url, impersonate=impersonate, proxy=proxy)
+ return result
+
+
+@router.post(
+ "/v1/tls/rotate",
+ tags=["Advanced"],
+ summary="Try multiple browser fingerprints until one succeeds",
+)
+async def tls_rotate(
+ url: str = Body(...),
+ proxy: str = Body(""),
+) -> dict[str, Any]:
+ """Try multiple browser fingerprints until one succeeds (anti-fingerprint rotation)."""
+ from tls_fingerprint import TLSScraper
+
+ s = TLSScraper()
+ if not s.is_available():
+ return {"success": False, "error": "curl_cffi not installed. Run: pip install curl_cffi"}
+ result = await s.fetch_with_rotation(url, proxy=proxy)
+ return result
+
+
+@router.post(
+ "/v1/camoufox/fetch", tags=["Advanced"], summary="Fetch with Camoufox anti-detection Firefox"
+)
+async def camoufox_fetch(
+ url: str = Body(...),
+ profile: str = Body("chrome_windows"),
+ wait_selector: str = Body(""),
+ proxy: str = Body(""),
+) -> dict[str, Any]:
+ """Fetch a URL using Camoufox (Firefox anti-detection browser).
+
+ Camoufox patches Firefox at the source level for maximum stealth.
+ This is more effective than Playwright for sites with advanced
+ fingerprinting (DataDome, PerimeterX, advanced Cloudflare).
+ """
+ from camoufox_integration import CamoufoxBrowser
+
+ b = CamoufoxBrowser()
+ if not b.is_available():
+ return {
+ "success": False,
+ "error": "camoufox not installed. Run: pip install camoufox && python -m camoufox fetch",
+ }
+ result = await b.fetch(url, profile=profile, wait_selector=wait_selector, proxy=proxy)
+ return result
+
+
+@router.post(
+ "/v1/graphql/discover", tags=["Advanced"], summary="Discover GraphQL endpoints for a site"
+)
+async def graphql_discover(base_url: str = Body(...)) -> dict[str, Any]:
+ """Auto-discover GraphQL endpoints for a website."""
+ from graphql_discovery import GraphQLDiscovery
+
+ g = GraphQLDiscovery()
+ found = await g.discover(base_url)
+ return {"success": True, "data": {"found": found, "count": len(found)}}
+
+
+@router.post(
+ "/v1/graphql/introspect", tags=["Advanced"], summary="Run GraphQL introspection on an endpoint"
+)
+async def graphql_introspect(endpoint: str = Body(...)) -> dict[str, Any]:
+ """Run GraphQL introspection query against a discovered endpoint."""
+ from graphql_discovery import GraphQLDiscovery
+
+ g = GraphQLDiscovery()
+ result = await g.introspect(endpoint)
+ return {"success": "error" not in result, "data": result}
+
+
+@router.post("/v1/graphql/query", tags=["Advanced"], summary="Execute a GraphQL query")
+async def graphql_query(
+ endpoint: str = Body(...),
+ query: str = Body(...),
+ variables: dict[str, Any] | None = Body(None),
+) -> dict[str, Any]:
+ """Execute a GraphQL query against a discovered endpoint."""
+ from graphql_discovery import GraphQLDiscovery
+
+ g = GraphQLDiscovery()
+ result = await g.query(endpoint, query, variables)
+ return {"success": "error" not in result, "data": result}
+
+
+@router.post(
+ "/v1/schema/extract",
+ tags=["Advanced"],
+ summary="Extract Schema.org structured data from a page",
+)
+async def schema_extract(url: str = Body(...)) -> dict[str, Any]:
+ """Extract Schema.org/JSON-LD/Microdata/RDFa structured data from a URL.
+ Most modern sites embed structured data โ extract it directly instead of
+ scraping HTML. 100x faster and more accurate."""
+ from schema_extraction import SchemaExtractor
+
+ client = await get_client()
+ resp = await client.get(url, timeout=30)
+ e = SchemaExtractor()
+ result = e.extract_all(resp.text)
+ return {"success": True, "data": result}
+
+
+@router.post(
+ "/v1/schema/extract-html",
+ tags=["Advanced"],
+ summary="Extract Schema.org structured data from raw HTML",
+)
+async def schema_extract_html(html: str = Body(...)) -> dict[str, Any]:
+ """Extract Schema.org/JSON-LD/Microdata/RDFa from a raw HTML string.
+ Useful when you've already fetched the page and want to extract structured data."""
+ from schema_extraction import SchemaExtractor
+
+ e = SchemaExtractor()
+ result = e.extract_all(html)
+ return {"success": True, "data": result}
+
+
+@router.post(
+ "/v1/ws/scrape",
+ tags=["Advanced"],
+ summary="Scrape data from a WebSocket endpoint",
+)
+async def ws_scrape(
+ url: str = Body(...),
+ max_messages: int = Body(100),
+ timeout: int = Body(30),
+ message_filter: str = Body(""),
+) -> dict[str, Any]:
+ """Connect to a WebSocket and capture the data stream.
+ Useful for SPAs and real-time apps that load data via WebSocket."""
+ from websocket_scraper import WebSocketScraper
+
+ s = WebSocketScraper(max_messages=max_messages, timeout=timeout)
+ result = await s.scrape_websocket(url, message_filter=message_filter)
+ return {"success": result.get("success", False), "data": result}
+
+
+@router.post(
+ "/v1/sse/scrape",
+ tags=["Advanced"],
+ summary="Scrape data from a Server-Sent Events endpoint",
+)
+async def sse_scrape(
+ url: str = Body(...),
+ max_events: int = Body(50),
+ timeout: int = Body(30),
+ event_filter: str = Body(""),
+) -> dict[str, Any]:
+ """Connect to a Server-Sent Events endpoint and capture events."""
+ from websocket_scraper import WebSocketScraper
+
+ s = WebSocketScraper(timeout=timeout)
+ result = await s.scrape_sse(url, event_filter=event_filter, max_events=max_events)
+ return {"success": result.get("success", False), "data": result}
+
+
+@router.post(
+ "/v1/cookies/warm",
+ tags=["Advanced"],
+ summary="Warm cookies for a domain by browsing legitimate pages",
+)
+async def warm_cookies(
+ target_domain: str = Body(...),
+ pages_to_visit: int = Body(3),
+) -> dict[str, Any]:
+ """Pre-age cookies for a domain to bypass anti-bot detection.
+ Visits legitimate pages first to build realistic browsing history."""
+ from cookie_warmer import CookieWarmer
+
+ w = CookieWarmer()
+ result = await w.warm_for_site(target_domain, pages_to_visit=pages_to_visit)
+ return {"success": result.get("success", False), "data": result}
+
+
+@router.get(
+ "/v1/cookies/sessions",
+ tags=["Advanced"],
+ summary="List all warmed cookie sessions",
+)
+async def list_cookie_sessions() -> dict[str, Any]:
+ from cookie_warmer import CookieWarmer
+
+ w = CookieWarmer()
+ return {"success": True, "data": w.list_sessions()}
+
+
+@router.post(
+ "/v1/pdf/extract",
+ tags=["Advanced"],
+ summary="Extract tables and text from a PDF",
+)
+async def extract_pdf(
+ pdf_url: str = Body(...),
+ method: str = Body("pdfplumber"),
+) -> dict[str, Any]:
+ """Download a PDF and extract structured tables and text."""
+ from pdf_extractor import PDFTableExtractor
+
+ client = await get_client()
+ try:
+ resp = await client.get(pdf_url, timeout=60)
+ if not resp.is_success:
+ return {
+ "success": False,
+ "error": f"Failed to download: HTTP {resp.status_code}",
+ }
+ e = PDFTableExtractor()
+ result = e.extract(resp.content, method=method)
+ return {"success": "error" not in result, "data": result}
+ except (httpx.HTTPError, httpx.RequestError) as e:
+ return {"success": False, "error": str(e)[:300]}
+
+
+@router.post(
+ "/v1/ocr/extract",
+ tags=["Advanced"],
+ summary="Extract text from an image using Tesseract",
+)
+async def ocr_extract(
+ image_url: str = Body(""),
+ image_base64: str = Body(""),
+) -> dict[str, Any]:
+ """Extract text from an image via URL or base64."""
+ from ocr_extractor import ImageOCR
+
+ o = ImageOCR()
+ if image_url:
+ result = await o.extract_from_url(image_url)
+ elif image_base64:
+ data = base64.b64decode(image_base64.split(",", 1)[-1])
+ result = o.extract_from_bytes(data)
+ else:
+ return {"success": False, "error": "Provide image_url or image_base64"}
+ return result
+
+
+@router.post(
+ "/v1/dedup/check",
+ tags=["Advanced"],
+ summary="Check if content is a near-duplicate using SimHash",
+)
+async def check_duplicate(
+ text: str = Body(...),
+ threshold: float = Body(0.85),
+) -> dict[str, Any]:
+ """Check if text is a near-duplicate of recently seen content using SimHash."""
+ from dedup import SimHash
+
+ h = SimHash.hash(text)
+ return {
+ "success": True,
+ "data": {
+ "hash": h,
+ "text_length": len(text),
+ "threshold": threshold,
+ },
+ }
+
+
+@router.get(
+ "/v1/behavior/simulate",
+ tags=["Advanced"],
+ summary="Generate human-like behavior patterns for testing",
+)
+async def get_behavior_simulation(action: str = "mouse") -> dict[str, Any]:
+ """Generate realistic human behavior patterns (mouse path, scroll, typing, etc.)"""
+ from behavioral_biometrics import behavior
+
+ if action == "mouse":
+ path = behavior.mouse_path((0, 0), (800, 600))
+ elif action == "scroll":
+ path = behavior.scroll_pattern(3000)
+ elif action == "typing":
+ path = behavior.typing_pattern("the quick brown fox jumps over")
+ elif action == "click":
+ return {
+ "success": True,
+ "data": {"delay_ms": behavior.click_decision_delay() * 1000},
+ }
+ else:
+ return {
+ "success": False,
+ "error": f"Unknown action: {action}. Use mouse|scroll|typing|click",
+ }
+ return {"success": True, "data": path}
diff --git a/routers/agency.py b/routers/agency.py
new file mode 100644
index 0000000..400a245
--- /dev/null
+++ b/routers/agency.py
@@ -0,0 +1,111 @@
+"""Pry โ Agency router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+from errors import NotFoundError
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Agency"])
+
+@router.post("/v1/agency/create", tags=["Agency"], summary="Create a white-label agency profile")
+async def create_agency_endpoint(
+ name: str = Body(...),
+ owner_email: str = Body(...),
+ custom_domain: str = Body(""),
+ brand_color: str = Body("#f59e0b"),
+ logo_url: str = Body(""),
+) -> dict[str, Any]:
+ """Create a white-label agency profile for reselling Pry.
+
+ Agencies get:
+ - Custom branding (colors, logo, domain)
+ - Client management with sub-accounts
+ - Usage analytics and quota management
+ - API key management for each client
+ """
+ from agency import create_agency
+
+ result = create_agency(name, owner_email, custom_domain, brand_color, logo_url)
+ return {"success": result["success"], "data": result}
+
+
+@router.get("/v1/agency/{agency_id}", tags=["Agency"], summary="Get agency profile")
+async def get_agency_endpoint(agency_id: str) -> dict[str, Any]:
+ """Get agency profile details."""
+ from agency import get_agency
+
+ result = get_agency(agency_id)
+ if not result:
+ raise NotFoundError(f"Agency not found: {agency_id}")
+ return {"success": True, "data": result}
+
+
+@router.put("/v1/agency/{agency_id}/branding", tags=["Agency"], summary="Update agency branding")
+async def update_branding(
+ agency_id: str,
+ name: str | None = Body(None),
+ brand_color: str | None = Body(None),
+ logo_url: str | None = Body(None),
+ custom_domain: str | None = Body(None),
+) -> dict[str, Any]:
+ """Update white-label branding for an agency."""
+ from agency import update_agency_branding
+
+ result = update_agency_branding(agency_id, name, brand_color, logo_url, custom_domain)
+ return {"success": result["success"], "data": result}
+
+
+@router.post("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account")
+async def create_client_endpoint(
+ agency_id: str,
+ client_name: str = Body(...),
+ client_email: str = Body(...),
+ monthly_quota: int = Body(10000),
+) -> dict[str, Any]:
+ """Create a client sub-account under an agency.
+
+ Each client gets their own API key and usage quota.
+ """
+ from agency import create_client
+
+ result = create_client(agency_id, client_name, client_email, monthly_quota)
+ return {"success": result["success"], "data": result}
+
+
+@router.get("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="List agency clients")
+async def list_clients_endpoint(agency_id: str) -> dict[str, Any]:
+ """List all clients under an agency."""
+ from agency import list_clients
+
+ clients = list_clients(agency_id)
+ return {"success": True, "data": {"clients": clients, "total": len(clients)}}
+
+
+@router.get("/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics")
+async def get_analytics(agency_id: str) -> dict[str, Any]:
+ """Get aggregate usage analytics for an agency."""
+ from agency import get_agency_analytics
+
+ result = get_agency_analytics(agency_id)
+ return {"success": True, "data": result}
+
+
+@router.get("/v1/client/{client_id}/quota", tags=["Agency"], summary="Check client quota usage")
+async def check_quota(client_id: str) -> dict[str, Any]:
+ """Check a client's current quota usage and remaining capacity."""
+ from agency import check_client_quota
+
+ result = check_client_quota(client_id)
+ return {"success": result["success"], "data": result}
diff --git a/routers/ai.py b/routers/ai.py
new file mode 100644
index 0000000..480ee10
--- /dev/null
+++ b/routers/ai.py
@@ -0,0 +1,41 @@
+"""Pry โ AI router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["AI"])
+
+@router.get("/v1/ai/gpt-manifest", tags=["AI"], summary="Get GPT Action manifest for ChatGPT")
+async def get_gpt_manifest() -> JSONResponse:
+ """Get the GPT Action manifest for ChatGPT integration.
+
+ Add this to your GPT configuration in ChatGPT to give it
+ web scraping capabilities through Pry.
+ """
+ from ai_plugin import get_gpt_action_manifest
+
+ return JSONResponse(content=get_gpt_action_manifest())
+
+
+@router.get("/v1/ai/mcp-config", tags=["AI"], summary="Get MCP server config for Claude/Cursor")
+async def get_mcp_config() -> JSONResponse:
+ """Get the MCP server configuration for Claude/Cursor.
+
+ Add this to your AI tool's MCP configuration file to let
+ Claude and Cursor scrape the web through Pry.
+ """
+ from ai_plugin import get_mcp_server_config
+
+ return JSONResponse(content={"success": True, "data": get_mcp_server_config()})
diff --git a/routers/alerts.py b/routers/alerts.py
new file mode 100644
index 0000000..1e5a969
--- /dev/null
+++ b/routers/alerts.py
@@ -0,0 +1,67 @@
+"""Pry โ Alerts router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Alerts"])
+
+@router.post("/v1/alert/send", tags=["Alerts"], summary="Send an alert to any channel")
+async def send_alert_endpoint(
+ channel: str = Body(...),
+ title: str = Body("Pry Alert"),
+ message: str = Body(...),
+ config: dict[str, Any] = Body(...),
+ severity: str = Body("info"),
+) -> dict[str, Any]:
+ """Send an alert to Slack, Discord, Teams, Telegram, SMS, or Email.
+
+ Config varies by channel:
+ - slack: {"webhook_url": "..."}
+ - discord: {"webhook_url": "..."}
+ - teams: {"webhook_url": "..."}
+ - telegram: {"bot_token": "...", "chat_id": "..."}
+ - sms: {"phone": "+1234567890", "twilio_sid": "...", "twilio_token": "...", "twilio_from": "..."}
+ - email: {"recipient": "user@example.com", "smtp_host": "...", "smtp_user": "..."}
+ """
+ from alerter import send_alert
+
+ result = await send_alert(channel, title, message, config, severity)
+ return {"success": result["success"], "data": result}
+
+
+@router.get("/v1/alert/channels", tags=["Alerts"], summary="List supported alert channels")
+async def list_channels() -> dict[str, Any]:
+ """List all supported alert channels and their config requirements."""
+ return {
+ "success": True,
+ "data": {
+ "channels": [
+ {"id": "slack", "name": "Slack", "config_fields": ["webhook_url"]},
+ {"id": "discord", "name": "Discord", "config_fields": ["webhook_url"]},
+ {"id": "teams", "name": "Microsoft Teams", "config_fields": ["webhook_url"]},
+ {"id": "telegram", "name": "Telegram", "config_fields": ["bot_token", "chat_id"]},
+ {
+ "id": "sms",
+ "name": "SMS (Twilio)",
+ "config_fields": ["phone", "twilio_sid", "twilio_token", "twilio_from"],
+ },
+ {
+ "id": "email",
+ "name": "Email",
+ "config_fields": ["recipient", "smtp_host", "smtp_user"],
+ },
+ ]
+ },
+ }
diff --git a/routers/analysis.py b/routers/analysis.py
new file mode 100644
index 0000000..ff8a00d
--- /dev/null
+++ b/routers/analysis.py
@@ -0,0 +1,199 @@
+"""Pry โ Analysis router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import asyncio
+import difflib
+import json
+import logging
+import re
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Body
+
+from client import get_client
+from deps import advanced, scraper
+from errors import ScrapeError
+from settings import settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Analysis"])
+
+@router.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs")
+async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]:
+ """Scrape two URLs and compare their content. Shows additions, deletions, changes."""
+ r1, r2 = await asyncio.gather(scraper.scrape(url1), scraper.scrape(url2))
+ c1, c2 = r1.get("content", ""), r2.get("content", "")
+ diff = list(
+ difflib.unified_diff(
+ c1.splitlines(keepends=True),
+ c2.splitlines(keepends=True),
+ fromfile=url1,
+ tofile=url2,
+ n=3,
+ )
+ )
+ return {
+ "success": True,
+ "data": {
+ "url1": url1,
+ "url2": url2,
+ "diff": diff[:200],
+ "changes": len(diff),
+ "identical": len(diff) == 0,
+ },
+ }
+
+
+@router.post("/v1/watch", tags=["Analysis"], summary="Watch a page for changes")
+async def watch_page(
+ url: str = Body(...), webhook: str = Body(...), interval: int = 3600, selector: str = ""
+) -> dict[str, Any]:
+ """Watch a page for changes. Accepts a webhook URL for notification.
+ First call registers the watch, subsequent calls compare and notify.
+ Firecrawl charges $99/mo for this feature."""
+ result = await scraper.scrape(url, {"bypass_cloudflare": True})
+ if result.get("status") != "ok":
+ raise ScrapeError("Could not scrape URL")
+ content = result.get("content", "")
+ diff_result = await advanced.track_diff(url, content)
+ # Fire webhook asynchronously on first registration (webhook support)
+ if webhook and diff_result["status"] == "registered":
+ asyncio.create_task(_fire_watch_webhook(webhook, url, diff_result))
+ return {
+ "success": True,
+ "data": {
+ "url": url,
+ "status": diff_result["status"],
+ "version": diff_result["version"],
+ "webhook": webhook,
+ "interval": interval,
+ "message": "Page registered for change tracking.",
+ },
+ }
+
+
+@router.post("/v1/summarize", tags=["Analysis"], summary="AI summarize scraped content")
+async def summarize(url: str = Body(...), max_words: int = 100) -> dict[str, Any]:
+ """Scrape + AI summarize using local Ollama. Free, private."""
+ result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30})
+ if result.get("status") != "ok":
+ raise ScrapeError(result.get("error", "Scrape failed"))
+ summary = await advanced.summarize(result.get("content", ""), max_words)
+ return {"success": True, "data": {"url": url, **summary}}
+
+
+@router.post("/v1/diff", tags=["Analysis"], summary="Track page changes over time")
+async def diff(url: str = Body(...), content: str | None = Body(None)) -> dict[str, Any]:
+ """Track page changes over time. Returns diff from last scrape."""
+ if content:
+ diff_result = await advanced.track_diff(url, content)
+ else:
+ result = await scraper.scrape(url)
+ if result.get("status") != "ok":
+ raise ScrapeError("Could not scrape URL")
+ diff_result = await advanced.track_diff(url, result.get("content", ""))
+ return {"success": True, "data": diff_result}
+
+
+@router.post("/v1/categorize", tags=["Analysis"], summary="AI-categorize scraped content")
+async def categorize(url: str = Body(...)) -> dict[str, Any]:
+ """AI-categorize scraped content into topic tags."""
+ result = await scraper.scrape(url)
+ if result.get("status") != "ok":
+ raise ScrapeError(result.get("error") or "Categorize failed")
+ tags = await advanced.categorize(result.get("content", ""))
+ kw = advanced.keyword_density(result.get("content", ""))
+ readability = advanced.readability(result.get("content", ""))
+ return {
+ "success": True,
+ "data": {"url": url, "tags": tags, "keywords": kw[:10], "readability": readability},
+ }
+
+
+@router.post("/v1/suggest", tags=["Analysis"], summary="AI-suggest schema fields for a URL")
+async def suggest_schema(data: dict[str, Any] = Body(...)) -> dict[str, Any]:
+ url = data.get("url", "")
+
+ result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30})
+ content = result.get("content", "")
+ html = ""
+ try:
+ client = await get_client()
+ resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15)
+ html = resp.text
+ except httpx.HTTPError:
+ logger.warning("suggest_schema_fetch_failed", extra={"url": url})
+ schema = {"_page_title": result.get("title", "")}
+ candidates: dict[str, str] = {}
+ for pattern, field in [
+ (r'["\']((?:price|cost|amount)[^"\']*)["\']', "price"),
+ (r'["\'](product-title|product-name|item-name)[^"\']*["\']', "name"),
+ (r'["\'](?:product-image|main-image|featured-image)[^"\']*["\']', "image"),
+ (r'["\'](?:description|product-desc|item-desc)[^"\']*["\']', "description"),
+ (r'["\'](?:rating|stars|review-score)[^"\']*["\']', "rating"),
+ (r'["\'](?:stock|availability|status)[^"\']*["\']', "stock"),
+ ]:
+ found = re.findall(pattern, html.lower())
+ if found:
+ candidates[field] = f'[class*="{found[0][:15]}"]'
+ if "name" not in candidates:
+ h1 = re.findall(r'Scrape engine health and performance metrics
+| Endpoint | Method | Status |
|---|---|---|
| /v1/scrape | POST | โ Active |
| /v1/crawl | POST | โ Active |
| /v1/automate | POST | โ Active |
| /v1/batch | POST | โ Active |
| /v1/stream | WebSocket | โ Active |
| /v1/run | POST | โ Active |
| FlareSolverr | Proxy | โ Connected |
Pry v3.0.0 โ Generated {datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")}
+""" + return HTMLResponse(content=html) diff --git a/routers/email.py b/routers/email.py new file mode 100644 index 0000000..204c384 --- /dev/null +++ b/routers/email.py @@ -0,0 +1,76 @@ +"""Pry โ Email router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Email"]) + +@router.post("/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)") +async def scrape_email( + subject: str = Body(""), + body: str = Body(""), + sender: str = Body(""), +) -> dict[str, Any]: + """Extract structured data from an email subject and body. + + Auto-classifies as: order_confirmation, invoice, receipt, + shipping_notification, subscription, or other. + + Extracts: order numbers, amounts, tracking numbers, dates, addresses. + """ + from email_scraper import extract_email_data + + result = extract_email_data(subject, body, sender) + return {"success": True, "data": result} + + +@router.post("/v1/email/gmail", tags=["Email"], summary="Fetch and extract data from Gmail inbox") +async def fetch_gmail( + access_token: str = Body(...), + max_results: int = Body(20), + query: str = Body(""), + since_days: int = Body(7), +) -> dict[str, Any]: + """Connect to Gmail and extract structured data from emails. + + Requires a Gmail OAuth2 access token with scope: + https://www.googleapis.com/auth/gmail.readonly + + Extracts order confirmations, invoices, receipts, shipping notifications. + """ + from email_scraper import fetch_gmail_emails + + result = await fetch_gmail_emails(access_token, max_results, query, since_days) + return {"success": result.get("success", False), "data": result} + + +@router.post("/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox") +async def fetch_outlook( + access_token: str = Body(...), + max_results: int = Body(20), + query: str = Body(""), + since_days: int = Body(7), +) -> dict[str, Any]: + """Connect to Outlook/Office 365 and extract structured data from emails. + + Requires a Microsoft Graph OAuth2 access token with scope: + https://graph.microsoft.com/Mail.Read + + Extracts order confirmations, invoices, receipts, shipping notifications. + """ + from email_scraper import fetch_outlook_emails + + result = await fetch_outlook_emails(access_token, max_results, query, since_days) + return {"success": result.get("success", False), "data": result} diff --git a/routers/enrichment.py b/routers/enrichment.py new file mode 100644 index 0000000..bbc9ab9 --- /dev/null +++ b/routers/enrichment.py @@ -0,0 +1,65 @@ +"""Pry โ Enrichment router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Body + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Enrichment"]) + +@router.post( + "/v1/enrich", + tags=["Enrichment"], + summary="Enrich scraped data with company info, tech stack, social profiles", +) +async def enrich_data( + url: str = Body(...), + include_tech_stack: bool = Body(True), + include_social: bool = Body(True), + include_company: bool = Body(True), +) -> dict[str, Any]: + """Enrich a URL with supplemental business intelligence. + + Returns: + - Tech stack detection (CMS, framework, CDN, analytics, payments) + - Social media profiles (Twitter, LinkedIn, Facebook, Instagram, GitHub, etc.) + - Company information (email, phone, address, founded year, team size) + """ + from enrichment import enrich_url + + result = await enrich_url(url) + filtered: dict[str, Any] = {"url": url} + if include_tech_stack: + filtered["tech_stack"] = result.get("tech_stack") + if include_social: + filtered["social_profiles"] = result.get("social_profiles") + if include_company: + filtered["company_info"] = result.get("company_info") + + return {"success": True, "data": filtered} + + +@router.post( + "/v1/enrich/tech-stack", tags=["Enrichment"], summary="Detect technologies used on a website" +) +async def detect_tech(url: str = Body(...)) -> dict[str, Any]: + """Detect what technologies a website is built with. + + Detects: CMS (WordPress, Shopify, Wix), frameworks (Next.js, Django, Rails), + frontend (React, Vue, Angular), CDN (Cloudflare, Fastly), analytics (GA, Hotjar), + payments (Stripe, PayPal). + """ + from enrichment import enrich_url + + result = await enrich_url(url) + return {"success": True, "data": result.get("tech_stack", {})} diff --git a/routers/execute.py b/routers/execute.py new file mode 100644 index 0000000..112e4a4 --- /dev/null +++ b/routers/execute.py @@ -0,0 +1,44 @@ +"""Pry โ Execute router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import InvalidRequestError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Execute"]) + +@router.post("/v1/run", tags=["Execute"], summary="Execute a Pryfile") +async def run_pryfile(path: str = Body("pry.yml")) -> dict[str, Any]: + from pryfile import Pryfile + + resolved = _safe_pryfile_path(path) + pf = Pryfile(resolved) + results = await pf.run_all(scraper) + return {"success": True, "data": {"jobs": results, "total": len(results)}} + + +def _safe_pryfile_path(path: str) -> str: + """Resolve and validate Pryfile path โ prevent directory traversal.""" + resolved = Path(path).resolve() + allowed = Path.cwd().resolve() + try: + resolved.relative_to(allowed) + except ValueError as e: + raise InvalidRequestError(f"Path must be inside {allowed}") from e + if not resolved.is_file(): + raise InvalidRequestError(f"File not found: {resolved}") + return str(resolved) diff --git a/routers/export.py b/routers/export.py new file mode 100644 index 0000000..5b42e48 --- /dev/null +++ b/routers/export.py @@ -0,0 +1,64 @@ +"""Pry โ Export router (remaining api.py routes). + +Auto-extracted from api.py during the router-split refactor. +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC + +from __future__ import annotations + +import html +import logging +from datetime import UTC +from typing import Any + +from fastapi import APIRouter, Body + +from deps import scraper +from errors import ScrapeError + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Export"]) + +@router.post("/v1/export", tags=["Export"], summary="Export scraped content in multiple formats") +async def export_data(url: str = Body(...), format: str = "json") -> dict[str, Any]: + """Export scraped content in multiple formats: json, csv, txt, rss. + Firecrawl only does markdown.""" + result = await scraper.scrape(url) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Export failed") + content = result.get("content", "") + title = result.get("title", url) + + if format == "json": + return { + "success": True, + "data": {"url": url, "title": title, "content": content, "format": "json"}, + } + elif format == "csv": + import csv + import io + + buf = io.StringIO() + w = csv.writer(buf) + w.writerow(["url", "title", "content"]) + w.writerow([url, title, content[:50000]]) + return {"success": True, "data": {"csv": buf.getvalue(), "format": "csv"}} + elif format == "rss": + from datetime import datetime + + safe_title = html.escape(title) + safe_url = html.escape(url) + rss = f""" +Share expired.
", 404) + st, sc, su = html.escape(d["title"]), html.escape(d["content"][:100000]), html.escape(d["url"]) + return HTMLResponse( + f'{sc}"
+ )
+
+
+_shares: dict[str, dict[str, Any]] = {}
diff --git a/routers/stats.py b/routers/stats.py
new file mode 100644
index 0000000..f989d62
--- /dev/null
+++ b/routers/stats.py
@@ -0,0 +1,28 @@
+"""Pry โ Stats router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter
+
+from deps import automator, cache, ratelimiter
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Stats"])
+
+@router.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats")
+async def stats() -> dict[str, Any]:
+ return {
+ "cache": cache.stats(),
+ "rate_limiter": ratelimiter.get_stats(),
+ "sessions": automator.list_sessions(),
+ }
diff --git a/routers/structure.py b/routers/structure.py
new file mode 100644
index 0000000..6be264f
--- /dev/null
+++ b/routers/structure.py
@@ -0,0 +1,110 @@
+"""Pry โ Structure router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+from errors import InvalidRequestError, NotFoundError
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Structure"])
+
+@router.post(
+ "/v1/structure/check",
+ tags=["Structure"],
+ summary="Check if CSS selectors still match on a page",
+)
+async def structure_check(
+ url: str = Body(...),
+ selectors: list[dict[str, Any]] = Body(...),
+) -> dict[str, Any]:
+ """Check if CSS selectors still match on a page.
+
+ Use this to verify your scraper templates still work after
+ a site redesign. Returns per-selector match status.
+
+ Selectors format: [{"name": "title", "selector": "h1", "type": "css"}]
+ """
+ from structure_monitor import check_selectors
+
+ result = await check_selectors(url, selectors)
+ return {"success": "error" not in result, "data": result}
+
+
+@router.post(
+ "/v1/structure/monitor", tags=["Structure"], summary="Monitor page structure changes over time"
+)
+async def structure_monitor(
+ url: str = Body(...),
+ selectors: list[dict[str, Any]] = Body(...),
+ template_id: str = Body(""),
+) -> dict[str, Any]:
+ """Monitor page structure changes and detect broken selectors.
+
+ Compares current selector match status to previous checks.
+ Alerts when selectors stop matching (site redesign detected).
+
+ Use this as a pre-scrape health check for your templates.
+ """
+ from structure_monitor import monitor_page_structure
+
+ result = await monitor_page_structure(url, selectors, template_id)
+ return {"success": "error" not in result, "data": result}
+
+
+@router.post(
+ "/v1/structure/check-template",
+ tags=["Structure"],
+ summary="Verify a scraper template still works",
+)
+async def structure_check_template(
+ template_id: str = Body(...), url: str = Body("")
+) -> dict[str, Any]:
+ """Check if a pre-built scraper template still works against a URL.
+
+ Extracts the template's selectors and checks each one.
+ If selectors fail, the template needs updating.
+ """
+ from structure_monitor import monitor_page_structure
+ from template_engine import get_template
+
+ template = get_template(template_id)
+ if not template:
+ raise NotFoundError(f"Template not found: {template_id}")
+
+ schema = template.get("schema", {})
+ fields = schema.get("fields", [])
+ selectors = [
+ {
+ "name": f.get("name", f"field_{i}"),
+ "selector": f.get("selector", ""),
+ "type": "css" if f.get("type") != "xpath" else "xpath",
+ }
+ for i, f in enumerate(fields)
+ if f.get("selector")
+ ]
+
+ if not selectors:
+ raise InvalidRequestError("Template has no selectable fields")
+
+ # If no URL provided, try the template's site URL
+ if not url:
+ site = template.get("site", "")
+ url = (
+ f"https://www.{site}"
+ if site and not site.startswith("http")
+ else (site if site else "https://example.com")
+ )
+
+ result = await monitor_page_structure(url, selectors, template_id)
+ return {"success": "error" not in result, "data": result}
diff --git a/routers/system.py b/routers/system.py
new file mode 100644
index 0000000..781035d
--- /dev/null
+++ b/routers/system.py
@@ -0,0 +1,44 @@
+"""Pry โ System router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, Response
+from fastapi.responses import JSONResponse
+
+from observability import get_metrics_output
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["System"])
+
+@router.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False)
+async def metrics() -> Response:
+ """Expose Prometheus metrics for scraping."""
+ data, content_type = get_metrics_output()
+ return Response(content=data, media_type=content_type)
+
+
+@router.get(
+ "/openapi.json",
+ tags=["System"],
+ summary="OpenAPI spec for AI agent integration",
+ include_in_schema=False,
+)
+async def get_openapi() -> JSONResponse:
+ """Get the OpenAPI specification for AI agent integration.
+
+ Use this with ChatGPT GPT Actions, Claude MCP, or any AI agent
+ framework to let AI models scrape the web through Pry.
+ """
+ from ai_plugin import get_openapi_spec
+
+ spec = get_openapi_spec()
+ return JSONResponse(content=spec)
diff --git a/routers/training.py b/routers/training.py
new file mode 100644
index 0000000..38b829d
--- /dev/null
+++ b/routers/training.py
@@ -0,0 +1,121 @@
+"""Pry โ Training router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+from errors import NotFoundError
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Training"])
+
+@router.post(
+ "/v1/training/classify-license",
+ tags=["Training"],
+ summary="Classify content license for AI training",
+)
+async def classify_license_endpoint(text: str = Body(...)) -> dict[str, Any]:
+ """Classify the license of scraped content for AI training compliance.
+
+ Returns license type (CC0, CC-BY, MIT, Apache, GPL, Proprietary, Fair Use),
+ tier (permissive/copyleft/restrictive/conditional), and confidence.
+ """
+ from training_data import classify_license
+
+ result = classify_license(text)
+ return {"success": True, "data": result}
+
+
+@router.post("/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content")
+async def clean_content(
+ text: str = Body(...),
+ strip_names: bool = Body(False),
+ strip_copyright: bool = Body(True),
+) -> dict[str, Any]:
+ """Strip PII and copyright content for AI training clean room.
+
+ Removes emails, phones, SSNs, credit cards, IPs, and (optionally) names.
+ Also strips copyright notices and near-verbatim copyright content.
+ """
+ from training_data import strip_copyright_verbatim, strip_pii
+
+ cleaned, pii_stats = strip_pii(text, preserve_names=not strip_names)
+ copyright_stats = {"blocks_removed": 0, "total_chars_removed": 0}
+
+ if strip_copyright:
+ cleaned, copyright_stats = strip_copyright_verbatim(cleaned)
+
+ return {
+ "success": True,
+ "data": {
+ "original_length": len(text),
+ "cleaned_length": len(cleaned),
+ "pii_removed": pii_stats,
+ "copyright_blocks_removed": copyright_stats["blocks_removed"],
+ "copyright_chars_removed": copyright_stats["total_chars_removed"],
+ "cleaned_content": cleaned[:5000] if len(cleaned) > 5000 else cleaned,
+ "truncated": len(cleaned) > 5000,
+ },
+ }
+
+
+@router.post(
+ "/v1/training/export",
+ tags=["Training"],
+ summary="Export a clean AI training dataset",
+)
+async def export_dataset(
+ records: list[dict[str, Any]] = Body(...),
+ output_format: str = Body("jsonl"),
+ clean_room: bool = Body(True),
+ strip_names: bool = Body(False),
+) -> dict[str, Any]:
+ """Export scraped content as a clean AI training dataset.
+
+ Each record should have: content, url, and optional metadata.
+
+ Features:
+ - Per-record provenance tracking (source URL, timestamp, extraction method)
+ - PII stripping (email, phone, SSN, CC, IP)
+ - Copyright verbatim text removal
+ - License classification
+ - Compliance report generation
+ """
+ from training_data import export_training_dataset
+
+ result = export_training_dataset(
+ records=records,
+ format=output_format, # type: ignore
+ clean_room=clean_room,
+ strip_names=strip_names,
+ )
+ return {"success": result["success"], "data": result}
+
+
+@router.get(
+ "/v1/training/compliance/{dataset_id}",
+ tags=["Training"],
+ summary="Generate compliance report for a dataset",
+)
+async def compliance_report(dataset_id: str) -> dict[str, Any]:
+ """Generate a compliance report for an exported training dataset.
+
+ Report includes: data provenance, PII/copyright removal stats,
+ license classification, and legal recommendations.
+ """
+ from training_data import generate_compliance_report
+
+ result = generate_compliance_report(dataset_id)
+ if "error" in result:
+ raise NotFoundError(result["error"])
+ return {"success": True, "data": result}
diff --git a/routers/transform.py b/routers/transform.py
new file mode 100644
index 0000000..d9d72be
--- /dev/null
+++ b/routers/transform.py
@@ -0,0 +1,68 @@
+"""Pry โ Transform router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+from deps import scraper
+from errors import InvalidRequestError, ScrapeError
+from pryextras import TransformEngine
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Transform"])
+
+@router.post("/v1/transform", tags=["Transform"], summary="Transform data to multiple formats")
+async def transform_data(
+ data: list[dict[str, Any]] = Body(...), format: str = "csv"
+) -> dict[str, Any]:
+ te = TransformEngine()
+ if format == "csv":
+ result = te.to_csv(data)
+ elif format == "sql":
+ result = "\n".join(te.to_sql(row) for row in data)
+ elif format == "html":
+ result = te.to_html_table(data)
+ elif format == "markdown":
+ result = te.to_markdown_table(data)
+ else:
+ raise InvalidRequestError(f"Unknown format: {format}")
+ return {"success": True, "data": {"output": result, "format": format}}
+
+
+@router.post("/v1/pipe", tags=["Transform"], summary="Scrape and transform via data pipeline")
+async def data_pipeline(data: dict[str, Any] = Body(...)) -> dict[str, Any]:
+ url = data.get("url", "")
+ transform = data.get("transform", "json")
+ result = await scraper.scrape(url, {"bypass_cloudflare": True})
+ if result.get("status") != "ok":
+ raise ScrapeError(result.get("error") or "Pipeline failed")
+ content = result.get("content", "")
+ title = result.get("title", url)
+ if transform == "sql":
+ from pryextras import TransformEngine
+
+ te = TransformEngine()
+ output = te.to_sql({"url": url, "title": title, "content": content[:50000]})
+ elif transform == "csv":
+ import csv
+ import io
+
+ buf = io.StringIO()
+ w = csv.writer(buf)
+ w.writerow(["url", "title", "content"])
+ w.writerow([url, title, content[:50000]])
+ output = buf.getvalue()
+ else:
+ output = json.dumps({"url": url, "title": title, "content": content[:100000]}, indent=2)
+ return {"success": True, "data": {"url": url, "output": output, "format": transform}}
diff --git a/routers/untagged.py b/routers/untagged.py
new file mode 100644
index 0000000..0a74ed3
--- /dev/null
+++ b/routers/untagged.py
@@ -0,0 +1,30 @@
+"""Pry โ Untagged router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect
+
+from pryextras import streams
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Untagged"])
+
+@router.websocket("/v1/stream/{job_id}")
+async def stream_endpoint(websocket: WebSocket, job_id: str = "live") -> None:
+ await websocket.accept()
+ streams.register(job_id, websocket)
+ try:
+ while True:
+ data = await websocket.receive_text()
+ await streams.broadcast(job_id, {"echo": data})
+ except WebSocketDisconnect:
+ streams.unregister(job_id, websocket)
diff --git a/routers/vision.py b/routers/vision.py
new file mode 100644
index 0000000..6a3fd01
--- /dev/null
+++ b/routers/vision.py
@@ -0,0 +1,221 @@
+"""Pry โ Vision router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import base64
+import logging
+import os
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Body
+
+from client import get_client
+from deps import automator
+from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError
+from settings import settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Vision"])
+
+@router.post("/v1/vision", tags=["Vision"], summary="Analyze an image with a free vision model")
+async def vision(
+ question: str = Body("Describe what is visible in this image.", embed=True),
+ image: str | None = Body(None, embed=True),
+ url: str | None = Body(None, embed=True),
+ file_path: str | None = Body(None, embed=True),
+ model: str | None = Body(None, embed=True),
+ max_tokens: int = Body(800, embed=True),
+ session_id: str | None = Body(None, embed=True),
+ no_fallback: bool = Body(False, embed=True),
+) -> dict[str, Any]:
+ """Analyze an image with a free vision model.
+
+ Provide ONE of: image (base64 or data URI), url (auto-screenshot),
+ or file_path (local PNG/JPG).
+
+ Auto-falls-back across 5 free OpenRouter vision models if the
+ requested one is rate-limited.
+ """
+ try:
+ # 1. Resolve the image bytes
+ if url:
+ # Auto-screenshot via playwright
+ r = await automator.run_steps(
+ steps=[{"action": "navigate", "url": url}, {"action": "screenshot"}],
+ session_id=session_id,
+ )
+ image_b64 = None
+ for step in r.get("steps", []):
+ if step.get("action") == "screenshot":
+ image_b64 = step.get("screenshot")
+ if not image_b64:
+ raise ScrapeError("screenshot returned empty")
+ elif file_path:
+ p = os.path.expanduser(file_path)
+ if not os.path.isfile(p):
+ raise InvalidRequestError(f"file not found: {file_path}")
+ with open(p, "rb") as f:
+ image_b64 = base64.b64encode(f.read()).decode("ascii")
+ elif image:
+ # Strip data URI prefix if present
+ image_b64 = image.split(",", 1)[1] if image.startswith("data:") else image
+ else:
+ raise InvalidRequestError("must provide one of: image, url, file_path")
+
+ # 2. Build the fallback chain
+ if model:
+ models_to_try = [model]
+ if not no_fallback:
+ models_to_try += [m for m in VISION_MODELS_FALLBACK if m != model]
+ else:
+ models_to_try = list(VISION_MODELS_FALLBACK)
+
+ # 3. Try each model until one succeeds
+ attempts = []
+ for m in models_to_try:
+ try:
+ text, used, err = await _call_vision_api(image_b64, question, m, max_tokens)
+ if err:
+ attempts.append({"model": m, "error": err})
+ continue
+ return {
+ "success": True,
+ "data": {
+ "answer": text,
+ "model_used": used,
+ "question": question,
+ "image_size_bytes": len(image_b64) * 3 // 4,
+ "attempts": attempts,
+ },
+ }
+ except httpx.HTTPStatusError as e:
+ attempts.append(
+ {
+ "model": m,
+ "error": f"http {e.response.status_code}",
+ "body": e.response.text[:200],
+ }
+ )
+ except OSError as e:
+ attempts.append({"model": m, "error": str(e)})
+
+ raise ExternalServiceError(
+ "all vision models failed",
+ details={"attempts": attempts},
+ )
+ except PryError:
+ raise
+ except OSError as e:
+ raise ExternalServiceError(str(e)) from e
+
+
+@router.get(
+ "/v1/vision/models", tags=["Vision"], summary="List free vision-capable models on OpenRouter"
+)
+async def vision_models() -> dict[str, Any]:
+ """List free vision-capable models on OpenRouter."""
+ try:
+ client = await get_client()
+ resp = await client.get(
+ "https://openrouter.ai/api/v1/models", headers={"User-Agent": "pry/3.0"}, timeout=15
+ )
+ resp.raise_for_status()
+ data = resp.json()
+ except httpx.HTTPError as e:
+ raise ExternalServiceError(str(e)) from e
+
+ free = []
+ for m in data.get("data", []):
+ arch = m.get("architecture", {})
+ if "image" not in arch.get("input_modalities", []):
+ continue
+ pricing = m.get("pricing", {})
+ try:
+ pp = float(pricing.get("prompt", "1") or 1)
+ cp = float(pricing.get("completion", "1") or 1)
+ except (httpx.HTTPError, httpx.RequestError):
+ continue
+ if pp == 0 and cp == 0:
+ free.append(
+ {
+ "id": m["id"],
+ "name": m.get("name", ""),
+ "context": arch.get("context_length"),
+ "description": (m.get("description") or "")[:120],
+ }
+ )
+ return {"success": True, "data": {"free_models": free, "count": len(free)}}
+
+
+VISION_MODELS_FALLBACK = [
+ "google/gemma-4-31b-it:free",
+ "google/gemma-4-26b-a4b-it:free",
+ "nvidia/nemotron-nano-12b-v2-vl:free",
+ "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free",
+ "openrouter/free",
+]
+
+
+async def _call_vision_api(
+ image_b64: str, question: str, model: str, max_tokens: int = 800
+) -> tuple[str | None, str | None, str | None]:
+ """Single async vision call. Returns (text, model_used) or raises."""
+ key = _get_or_key()
+ if not key:
+ return None, None, "OPENROUTER_API_KEY not set"
+
+ # Accept either data URI or raw base64
+ data_uri = image_b64 if image_b64.startswith("data:") else f"data:image/png;base64,{image_b64}"
+
+ payload = {
+ "model": model,
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": question},
+ {"type": "image_url", "image_url": {"url": data_uri}},
+ ],
+ }
+ ],
+ "max_tokens": max_tokens,
+ }
+
+ client = await get_client()
+ resp = await client.post(
+ "https://openrouter.ai/api/v1/chat/completions",
+ json=payload,
+ headers={
+ "Authorization": f"Bearer {key}",
+ "Content-Type": "application/json",
+ "HTTP-Referer": "https://pry.local",
+ "X-Title": "pry-vision",
+ },
+ timeout=60,
+ )
+ resp.raise_for_status()
+ body = resp.json()
+ return body["choices"][0]["message"]["content"], model, None
+
+
+def _get_or_key() -> str | None:
+ """Pull OPENROUTER_API_KEY from ~/.hermes/.env or env."""
+ key = settings.openrouter_api_key
+ if key:
+ return key
+ env_path = os.path.expanduser("~/.hermes/.env")
+ if not os.path.isfile(env_path):
+ return None
+ with open(env_path) as f:
+ for line in f:
+ if line.startswith("OPENROUTER_API_KEY="):
+ return line.split("=", 1)[1].strip().strip('"').strip("'")
+ return None
diff --git a/routers/webhooks.py b/routers/webhooks.py
new file mode 100644
index 0000000..0b8acae
--- /dev/null
+++ b/routers/webhooks.py
@@ -0,0 +1,46 @@
+"""Pry โ Webhooks router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Webhooks"])
+
+@router.post(
+ "/v1/webhooks/test",
+ tags=["Webhooks"],
+ summary="Test webhook delivery",
+)
+async def test_webhook(
+ url: str = Body(...),
+ payload: dict[str, Any] | None = Body(None),
+) -> dict[str, Any]:
+ """Test webhook delivery to a URL with HMAC signing."""
+ from webhook_delivery import WebhookDelivery
+
+ w = WebhookDelivery()
+ return await w.deliver(url, payload or {"test": True, "timestamp": "now"}, "test.event")
+
+
+@router.get(
+ "/v1/webhooks/dead-letter",
+ tags=["Webhooks"],
+ summary="Get failed webhook deliveries",
+)
+async def get_dead_letter() -> dict[str, Any]:
+ """Get the dead letter queue of failed webhook deliveries."""
+ from webhook_delivery import WebhookDelivery
+
+ w = WebhookDelivery()
+ return {"success": True, "data": w.retry_dead_letter()}
diff --git a/routers/x402.py b/routers/x402.py
new file mode 100644
index 0000000..56cf02e
--- /dev/null
+++ b/routers/x402.py
@@ -0,0 +1,153 @@
+"""Pry โ x402 router (remaining api.py routes).
+
+Auto-extracted from api.py during the router-split refactor.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from fastapi import APIRouter, Body
+
+from errors import InvalidRequestError
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["x402"])
+
+@router.get("/v1/x402/pricing", tags=["x402"], summary="Get x402 pricing for all paid operations")
+async def x402_pricing() -> dict[str, Any]:
+ """Get the price list for pay-per-scrape operations."""
+ from x402 import X402Handler
+
+ return {"success": True, "data": X402Handler().get_stats()}
+
+
+@router.post("/v1/x402/payment", tags=["x402"], summary="Create a x402 payment request")
+async def x402_payment_request(
+ operation: str = Body(...),
+ metadata: dict[str, Any] | None = Body(None),
+) -> dict[str, Any]:
+ """Create a x402 payment request for a paid operation.
+
+ Returns payment details (wallet, amount, asset) for the client to pay.
+ """
+ from x402 import create_payment_request
+
+ req = create_payment_request(operation, metadata)
+ return {"success": True, "data": req}
+
+
+@router.post("/v1/x402/verify", tags=["x402"], summary="Verify a x402 payment was settled")
+async def x402_verify_payment(
+ payment_id: str = Body(...),
+ tx_hash: str = Body(...),
+ network: str = Body(""),
+ asset: str = Body(""),
+ amount_usd: float = Body(0.0),
+) -> dict[str, Any]:
+ """Verify a x402 payment has been settled on-chain via facilitator router."""
+ from x402 import X402Handler
+
+ result = await X402Handler().verify_payment(
+ payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd
+ )
+ return {"success": result["verified"], "data": result}
+
+
+@router.post(
+ "/v1/x402/require-payment", tags=["x402"], summary="Generate a 402 Payment Required response"
+)
+async def x402_require_payment(
+ operation: str = Body(...),
+ metadata: dict[str, Any] | None = Body(None),
+) -> dict[str, Any]:
+ """Generate a 402 Payment Required response for a paid endpoint."""
+ from x402 import require_payment_legacy
+
+ return require_payment_legacy(operation, metadata)
+
+
+@router.post("/v1/x402/batch-payment", tags=["x402"], summary="Create a batch x402 payment")
+async def x402_batch_payment(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+ """Create a single x402 payment covering multiple operations.
+
+ Returns a PaymentRequired body with the combined amount. After paying,
+ submit the tx to POST /v1/x402/batch-verify.
+ """
+ from x402 import create_batch_payment
+
+ operations = payload.get("operations", [])
+ if not isinstance(operations, list):
+ raise InvalidRequestError("operations must be a list")
+ result = await create_batch_payment(operations)
+ if "error" in result:
+ return {"success": False, "error": result["error"]}
+ return {"success": True, "data": result}
+
+
+@router.post("/v1/x402/batch-verify", tags=["x402"], summary="Verify a batch x402 payment")
+async def x402_batch_verify(payload: dict[str, Any] = Body(...)) -> dict[str, Any]:
+ """Verify the on-chain payment for a batch and mark it paid."""
+ from x402 import verify_batch_payment
+
+ batch_id = payload.get("batch_id", "")
+ tx_hash = payload.get("tx_hash", "")
+ network = payload.get("network", "")
+ asset = payload.get("asset", "")
+ if not batch_id or not tx_hash:
+ raise InvalidRequestError("batch_id and tx_hash are required")
+ result = await verify_batch_payment(batch_id, tx_hash, network=network, asset=asset)
+ return {"success": result["verified"], "data": result}
+
+
+@router.post(
+ "/v1/x402/pay",
+ tags=["x402"],
+ summary="Process x402 payment and get access token",
+)
+async def x402_pay(
+ operation: str = Body(...),
+ tx_hash: str = Body(...),
+ payer_wallet: str = Body(...),
+ network: str = Body(""),
+ asset: str = Body(""),
+ amount_usd: float = Body(0.0),
+) -> dict[str, Any]:
+ """Process an x402 payment and return an access token.
+
+ Flow:
+ 1. User gets 402 from a paid endpoint
+ 2. User sends USDC to the wallet in the 402 response
+ 3. User calls this endpoint with the tx_hash
+ 4. Pry verifies the transaction through the facilitator router
+ 5. Returns access token (payment_id) to use in X-Payment-Hash header
+ """
+ from x402 import X402Handler
+
+ h = X402Handler()
+ payment_id = uuid.uuid4().hex[:12]
+ verify_result = await h.verify_payment(
+ payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd
+ )
+ settlement = None
+ if verify_result.get("verified"):
+ settlement = await h.settle_payment(
+ payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd
+ )
+ return {
+ "success": verify_result.get("verified", False),
+ "data": {
+ "payment_id": payment_id,
+ "tx_hash": tx_hash,
+ "verified": verify_result,
+ "settlement": settlement,
+ "payer_wallet": payer_wallet,
+ "use_in_header": {"X-Payment-ID": payment_id, "X-Payment-Hash": tx_hash},
+ },
+ }
diff --git a/settings.py b/settings.py
index e4ed772..8f432ff 100644
--- a/settings.py
+++ b/settings.py
@@ -25,7 +25,7 @@ class PrySettings(BaseSettings):
port: int = 8002
# Ollama LLM endpoint
- ollama_url: str = "http://100.100.18.18:11434"
+ ollama_url: str = "http://100.104.130.92:11434"
# FlareSolverr Cloudflare bypass endpoint
flaresolverr_url: str = "http://flaresolverr:8191/v1"
diff --git a/tests/test_api.py b/tests/test_api.py
index 54cb3b4..595c343 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -20,7 +20,7 @@ def client() -> TestClient:
def test_get_or_key_from_env(monkeypatch) -> None:
monkeypatch.setattr("api.settings.openrouter_api_key", "sk-test-key-123")
- from api import _get_or_key
+ from routers.vision import _get_or_key
key = _get_or_key()
assert key == "sk-test-key-123"
@@ -29,14 +29,14 @@ def test_get_or_key_from_env(monkeypatch) -> None:
def test_get_or_key_empty_when_not_set(monkeypatch) -> None:
monkeypatch.setattr("api.settings.openrouter_api_key", "")
monkeypatch.setattr("os.path.isfile", lambda p: False)
- from api import _get_or_key
+ from routers.vision import _get_or_key
key = _get_or_key()
assert key is None
def test_vision_models_fallback_list() -> None:
- from api import VISION_MODELS_FALLBACK
+ from routers.vision import VISION_MODELS_FALLBACK
assert len(VISION_MODELS_FALLBACK) >= 3
assert all("free" in m for m in VISION_MODELS_FALLBACK)
diff --git a/ultimate_scraper.py b/ultimate_scraper.py
index a4c8147..7ef4015 100644
--- a/ultimate_scraper.py
+++ b/ultimate_scraper.py
@@ -314,6 +314,11 @@ class UltimateScraper:
driver = uc.Chrome(headless=True, use_subprocess=True)
driver.get(url)
+ # Note: time.sleep is used inside a sync thread-pool task because
+ # undetected_chromedriver's blocking API runs in asyncio.to_thread.
+ # Replacing with asyncio.sleep would require restructuring the tier
+ # to be fully async; this sync sleep is bounded and does not block
+ # the event loop directly.
time.sleep(random.uniform(2, 4))
html = driver.page_source
driver.quit()