diff --git a/api.py b/api.py index 4bcbb42..e946326 100644 --- a/api.py +++ b/api.py @@ -388,51 +388,21 @@ class PryHttpMiddleware: app.add_middleware(PryHttpMiddleware) - - - - - - # ── Stats ── # ── Costing ── - - - - - - # ── Freshness ── - - - - - - - - - - - - - - # ── Parse (Documents) ── - - # ── Automate (Browser) ── - - # ── Vision — analyze an image via free OpenRouter vision models ─────────────── # Free models default to google/gemma-4-31b-it:free. Accepts: # - image (base64 data URI or raw base64) @@ -445,113 +415,39 @@ app.add_middleware(PryHttpMiddleware) # Auto-fallback: if primary model 429s, tries other free models in order. - - - - - - - - # ── Sessions ────────────────────────────────────────────────────────────────── - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # ── Email Inbox Scraping ── - - - - - - - - # ── Integrations ── - - - - - - # ── Alerts ── - - - - # ── Jobs ── # ── Config ── - - - - - - # ── Win 3: WebSocket streaming ── # ── Win 5: Batch from file + template ── - - # ── Win 6: Browser recorder ── - - - - - - # ── Win 8: Multi-format transform ── # ── Win 1: Pryfile execution ── - - - - # ── Win 10: Health dashboard HTML ── @@ -561,120 +457,42 @@ app.add_middleware(PryHttpMiddleware) # ── Win 2: Circuit Breaker ── - - - - # ── Win 3: Stable Extraction ── - - - - # ── Pipeline hooks ── - - - - - - # ── Win 4: Data Pipeline ── # ── Win 5: Shareable Results ── - - - - # ── Compliance ── - - - - # ── GDPR Compliance Portal ── - - - - - - - - - - - - - - - - # ── AI Training Data Pipeline ── - - - - - - - - # ── Monitoring (cron-based content change detection with AI judging) ── - - - - - - - - - - # ── Structure Monitor ── - - - - - - # ── Intelligence (Competitive Intelligence Engine) ── - - - - - - - - # ── Quality ── - - - - # ── Reconciliation ── - - - - # ── Auth ── # (split into routers/auth.py on the api-router-split refactor) @@ -733,201 +551,50 @@ app.include_router(webhooks_router) app.include_router(x402_router) - # ── Review ── - - - - - - - - - - - - # ── Commerce Sync ── - - - - # ── CRM Sync ── - - - - # ── Pipeline Builder Endpoints ── - - - - - - - - - - - - - - # ── Agency ── - - - - - - - - - - - - - - # ── SEO Content Change Monitor ── - - - - - - # ── Reports ── - - - - - - # ── Enrichment ── - - - - # ── AI Agent Integration ── - - - - - - - - - - - - - - - - - - - - - - - - - - # ── Proxy Provider & Affiliate Signup Flow ── - - - - - - - - - - - - - - - - # ── Advanced Scraping (TLS, GraphQL, Schema.org, WebSocket) ── - - - - - - - - - - - - - - - - - - - - # ── Advanced: Cookie Warming, PDF, OCR, Dedup, Behavior ── - - - - - - - - - - - - # ── x402 payment processing ── - - # ── Actor marketplace ── - - - - - - # ── Webhook delivery ── - - - - if __name__ == "__main__": uvicorn.run(app, host=settings.host, port=settings.port) diff --git a/pry_mcp/fallback_server.py b/pry_mcp/fallback_server.py index 3cd7091..19333bc 100644 --- a/pry_mcp/fallback_server.py +++ b/pry_mcp/fallback_server.py @@ -31,6 +31,7 @@ from pry_mcp.notifications import ( logger = logging.getLogger(__name__) + def make_fallback_server( base_url: str = DEFAULT_BASE_URL, tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]] diff --git a/pry_mcp/sdk_server.py b/pry_mcp/sdk_server.py index 3da9b3d..cd5183d 100644 --- a/pry_mcp/sdk_server.py +++ b/pry_mcp/sdk_server.py @@ -59,6 +59,7 @@ try: except ImportError: _has_mcp_sdk = False + def create_server() -> Any: """Create the MCP server (uses official SDK if available, fallback otherwise).""" if _has_mcp_sdk: diff --git a/routers/advanced.py b/routers/advanced.py index 76d2675..f7b3197 100644 --- a/routers/advanced.py +++ b/routers/advanced.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Advanced"]) + @router.post( "/v1/tls/impersonate", tags=["Advanced"], diff --git a/routers/agency.py b/routers/agency.py index 400a245..5a56b9b 100644 --- a/routers/agency.py +++ b/routers/agency.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Agency"]) + @router.post("/v1/agency/create", tags=["Agency"], summary="Create a white-label agency profile") async def create_agency_endpoint( name: str = Body(...), @@ -67,7 +68,9 @@ async def update_branding( return {"success": result["success"], "data": result} -@router.post("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account") +@router.post( + "/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account" +) async def create_client_endpoint( agency_id: str, client_name: str = Body(...), @@ -93,7 +96,9 @@ async def list_clients_endpoint(agency_id: str) -> dict[str, Any]: return {"success": True, "data": {"clients": clients, "total": len(clients)}} -@router.get("/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics") +@router.get( + "/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics" +) async def get_analytics(agency_id: str) -> dict[str, Any]: """Get aggregate usage analytics for an agency.""" from agency import get_agency_analytics diff --git a/routers/ai.py b/routers/ai.py index 480ee10..80975ee 100644 --- a/routers/ai.py +++ b/routers/ai.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["AI"]) + @router.get("/v1/ai/gpt-manifest", tags=["AI"], summary="Get GPT Action manifest for ChatGPT") async def get_gpt_manifest() -> JSONResponse: """Get the GPT Action manifest for ChatGPT integration. diff --git a/routers/alerts.py b/routers/alerts.py index 1e5a969..8e82aee 100644 --- a/routers/alerts.py +++ b/routers/alerts.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Alerts"]) + @router.post("/v1/alert/send", tags=["Alerts"], summary="Send an alert to any channel") async def send_alert_endpoint( channel: str = Body(...), diff --git a/routers/analysis.py b/routers/analysis.py index ff8a00d..8732588 100644 --- a/routers/analysis.py +++ b/routers/analysis.py @@ -27,6 +27,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Analysis"]) + @router.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs") async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]: """Scrape two URLs and compare their content. Shows additions, deletions, changes.""" diff --git a/routers/automation.py b/routers/automation.py index 8cceaf8..18e3ea7 100644 --- a/routers/automation.py +++ b/routers/automation.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Automation"]) + @router.post("/v1/automate", tags=["Automation"], summary="Execute browser automation steps") async def automate(request: AutomateRequest) -> dict[str, Any]: """Execute browser automation steps. Sessions persist for login flows.""" diff --git a/routers/batch.py b/routers/batch.py index aba2b53..b1e0001 100644 --- a/routers/batch.py +++ b/routers/batch.py @@ -20,7 +20,10 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Batch"]) -@router.post("/v1/batch-file", tags=["Batch"], summary="Batch scrape URLs from a file with templates") + +@router.post( + "/v1/batch-file", tags=["Batch"], summary="Batch scrape URLs from a file with templates" +) async def batch_from_file(request: BatchFileRequest) -> dict[str, Any]: bp = BatchProcessor() results = await bp.from_file( diff --git a/routers/circuit_breaker.py b/routers/circuit_breaker.py index 1076c35..20ff607 100644 --- a/routers/circuit_breaker.py +++ b/routers/circuit_breaker.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Circuit Breaker"]) + @router.post("/v1/breaker/status", tags=["Circuit Breaker"], summary="Get circuit breaker status") async def breaker_status() -> dict[str, Any]: return { diff --git a/routers/commerce.py b/routers/commerce.py index 8e001e0..b442d44 100644 --- a/routers/commerce.py +++ b/routers/commerce.py @@ -19,7 +19,10 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Commerce"]) -@router.post("/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify") + +@router.post( + "/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify" +) async def sync_commerce( platform: str = Body(...), products: list[dict[str, Any]] = Body(...), @@ -58,7 +61,9 @@ async def sync_commerce( return {"success": result["success"], "data": result} -@router.get("/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms") +@router.get( + "/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms" +) async def list_commerce_platforms() -> dict[str, Any]: """List supported e-commerce platforms and their credential requirements.""" return { diff --git a/routers/compliance.py b/routers/compliance.py index 4cb7755..5ff1a64 100644 --- a/routers/compliance.py +++ b/routers/compliance.py @@ -17,7 +17,10 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Compliance"]) -@router.post("/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL") + +@router.post( + "/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL" +) async def compliance_check(url: str = Body(...)) -> dict[str, Any]: """Run a full legal compliance check on a target URL. @@ -35,7 +38,9 @@ async def compliance_check(url: str = Body(...)) -> dict[str, Any]: return {"success": True, "data": result} -@router.get("/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation") +@router.get( + "/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation" +) async def compliance_docs() -> dict[str, Any]: """Get information about the compliance check endpoint.""" return { diff --git a/routers/config.py b/routers/config.py index d027b65..b4329d8 100644 --- a/routers/config.py +++ b/routers/config.py @@ -34,7 +34,9 @@ async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]: return {"success": True, "data": result} -@router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests") +@router.post( + "/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests" +) async def enable_tor() -> dict[str, Any]: """Enable Tor routing for all requests.""" result = settings.enable_tor() diff --git a/routers/costing.py b/routers/costing.py index 2f0a022..f07c35d 100644 --- a/routers/costing.py +++ b/routers/costing.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Costing"]) + @router.get("/v1/costing/dashboard", tags=["Costing"], summary="Get cost analytics dashboard") async def cost_dashboard() -> dict[str, Any]: """Get the full cost analytics dashboard. diff --git a/routers/crm.py b/routers/crm.py index 62ae33f..c7f4e22 100644 --- a/routers/crm.py +++ b/routers/crm.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["CRM"]) + @router.post("/v1/crm/sync", tags=["CRM"], summary="Sync scraped data to CRM") async def sync_crm( platform: str = Body(...), diff --git a/routers/dashboard.py b/routers/dashboard.py index 6a6f09e..3826933 100644 --- a/routers/dashboard.py +++ b/routers/dashboard.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Dashboard"]) + @router.get("/dashboard", tags=["Dashboard"], summary="Pry health and performance dashboard") async def dashboard() -> HTMLResponse: cache_stats = cache.stats() diff --git a/routers/email.py b/routers/email.py index 204c384..80b5ad2 100644 --- a/routers/email.py +++ b/routers/email.py @@ -17,7 +17,10 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Email"]) -@router.post("/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)") + +@router.post( + "/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)" +) async def scrape_email( subject: str = Body(""), body: str = Body(""), @@ -56,7 +59,9 @@ async def fetch_gmail( return {"success": result.get("success", False), "data": result} -@router.post("/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox") +@router.post( + "/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox" +) async def fetch_outlook( access_token: str = Body(...), max_results: int = Body(20), diff --git a/routers/enrichment.py b/routers/enrichment.py index bbc9ab9..9638a32 100644 --- a/routers/enrichment.py +++ b/routers/enrichment.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Enrichment"]) + @router.post( "/v1/enrich", tags=["Enrichment"], diff --git a/routers/execute.py b/routers/execute.py index 112e4a4..ee5e402 100644 --- a/routers/execute.py +++ b/routers/execute.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Execute"]) + @router.post("/v1/run", tags=["Execute"], summary="Execute a Pryfile") async def run_pryfile(path: str = Body("pry.yml")) -> dict[str, Any]: from pryfile import Pryfile diff --git a/routers/export.py b/routers/export.py index 5b42e48..1c2368c 100644 --- a/routers/export.py +++ b/routers/export.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Export"]) + @router.post("/v1/export", tags=["Export"], summary="Export scraped content in multiple formats") async def export_data(url: str = Body(...), format: str = "json") -> dict[str, Any]: """Export scraped content in multiple formats: json, csv, txt, rss. diff --git a/routers/extraction.py b/routers/extraction.py index dcde526..0f216b3 100644 --- a/routers/extraction.py +++ b/routers/extraction.py @@ -26,6 +26,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Extraction"]) + @router.post( "/v1/extract-table", tags=["Extraction"], summary="Extract HTML tables as structured data" ) @@ -122,7 +123,9 @@ async def analyze_seo(url: str = Body(...)) -> dict[str, Any]: } -@router.post("/v1/schema", tags=["Extraction"], summary="Extract Schema.org/JSON-LD structured data") +@router.post( + "/v1/schema", tags=["Extraction"], summary="Extract Schema.org/JSON-LD structured data" +) async def extract_schema(url: str = Body(...)) -> dict[str, Any]: """Extract Schema.org/JSON-LD structured data from a page.""" client = await get_client() @@ -211,7 +214,9 @@ async def extract_css( } -@router.post("/v1/extract/llm", tags=["Extraction"], summary="Extract with LLM + chunking strategies") +@router.post( + "/v1/extract/llm", tags=["Extraction"], summary="Extract with LLM + chunking strategies" +) async def extract_llm( url: str = Body(...), instruction: str = Body("Extract all key information from this content."), diff --git a/routers/freshness.py b/routers/freshness.py index 510da7b..738c3de 100644 --- a/routers/freshness.py +++ b/routers/freshness.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Freshness"]) + @router.post( "/v1/freshness/check", tags=["Freshness"], @@ -63,7 +64,9 @@ async def freshness_frequency( return {"success": True, "data": result} -@router.get("/v1/freshness/dashboard", tags=["Freshness"], summary="Get content staleness dashboard") +@router.get( + "/v1/freshness/dashboard", tags=["Freshness"], summary="Get content staleness dashboard" +) async def freshness_dashboard() -> dict[str, Any]: """Get the content staleness dashboard. diff --git a/routers/gdpr.py b/routers/gdpr.py index ae2b79a..7966ed9 100644 --- a/routers/gdpr.py +++ b/routers/gdpr.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["GDPR"]) + @router.post("/v1/gdpr/consent", tags=["GDPR"], summary="Record user consent for data processing") async def record_consent_endpoint( user_id: str = Body(...), diff --git a/routers/integrations.py b/routers/integrations.py index d5038b7..f20c294 100644 --- a/routers/integrations.py +++ b/routers/integrations.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Integrations"]) + @router.get( "/v1/destinations", tags=["Integrations"], diff --git a/routers/intelligence.py b/routers/intelligence.py index a954f43..4795c54 100644 --- a/routers/intelligence.py +++ b/routers/intelligence.py @@ -17,7 +17,10 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Intelligence"]) -@router.post("/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot") + +@router.post( + "/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot" +) async def record_intel_snapshot( competitor_id: str = Body(...), competitor_name: str = Body(...), diff --git a/routers/jobs.py b/routers/jobs.py index f84fa4b..4924d54 100644 --- a/routers/jobs.py +++ b/routers/jobs.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Jobs"]) + @router.get("/v1/job/{job_id}", tags=["Jobs"], summary="Get async job status and result") async def get_job(job_id: str) -> dict[str, Any]: """Get async job status and result.""" diff --git a/routers/marketplace.py b/routers/marketplace.py index 4dbe322..1c73c54 100644 --- a/routers/marketplace.py +++ b/routers/marketplace.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Marketplace"]) + @router.post( "/v1/actors/create", tags=["Marketplace"], diff --git a/routers/monitoring.py b/routers/monitoring.py index c249a3e..684ef07 100644 --- a/routers/monitoring.py +++ b/routers/monitoring.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Monitoring"]) + @router.post("/v1/monitor", tags=["Monitoring"], summary="Create a scheduled content monitor") async def create_monitor_endpoint( name: str = Body(...), diff --git a/routers/parsing.py b/routers/parsing.py index 64aba8a..151ea33 100644 --- a/routers/parsing.py +++ b/routers/parsing.py @@ -23,7 +23,10 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Parsing"]) -@router.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)") + +@router.post( + "/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)" +) async def parse_document(request: ParseRequest) -> dict[str, Any]: """Parse a document (PDF, DOCX, image, CSV, JSON) to text.""" try: diff --git a/routers/pipeline.py b/routers/pipeline.py index 9571eb3..cbba320 100644 --- a/routers/pipeline.py +++ b/routers/pipeline.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Pipeline"]) + @router.post("/v1/pipeline/hook", tags=["Pipeline"], summary="Register a hook function") async def register_hook( hook_point: str = Body(...), diff --git a/routers/pipelines.py b/routers/pipelines.py index 022bcca..dcd3b0d 100644 --- a/routers/pipelines.py +++ b/routers/pipelines.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Pipelines"]) + @router.get( "/v1/pipelines/steps", tags=["Pipelines"], summary="List all available pipeline step types" ) diff --git a/routers/proxy.py b/routers/proxy.py index 92fce5b..a16aa87 100644 --- a/routers/proxy.py +++ b/routers/proxy.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Proxy"]) + @router.get("/v1/proxy/providers", tags=["Proxy"], summary="List all available proxy providers") async def list_proxy_providers() -> dict[str, Any]: """List free + premium proxy providers with affiliate details.""" @@ -25,7 +26,9 @@ async def list_proxy_providers() -> dict[str, Any]: return {"success": True, "data": ProxyManager().list_providers()} -@router.post("/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider") +@router.post( + "/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider" +) async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]: """Get the affiliate signup URL for a proxy provider. @@ -39,7 +42,9 @@ async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]: return {"success": True, "data": {"signup_url": url, "provider": provider}} -@router.get("/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals") +@router.get( + "/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals" +) async def list_proxy_referrals() -> dict[str, Any]: """Return the curated proxy provider affiliate catalog (proxy_referrals.py). @@ -126,7 +131,9 @@ async def proxy_status() -> dict[str, Any]: } -@router.post("/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block") +@router.post( + "/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block" +) async def proxy_recommend(last_error: str = Body("")) -> dict[str, Any]: """After a scrape fails with anti-bot detection, get a recommendation for which premium proxy provider to sign up with.""" diff --git a/routers/quality.py b/routers/quality.py index bd2f472..a3638dc 100644 --- a/routers/quality.py +++ b/routers/quality.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Quality"]) + @router.post( "/v1/quality/check", tags=["Quality"], summary="Run data quality check on extraction results" ) diff --git a/routers/reconciliation.py b/routers/reconciliation.py index 20cacad..3fb6cfe 100644 --- a/routers/reconciliation.py +++ b/routers/reconciliation.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Reconciliation"]) + @router.post( "/v1/reconcile", tags=["Reconciliation"], diff --git a/routers/recorder.py b/routers/recorder.py index 67653fd..ca25d6b 100644 --- a/routers/recorder.py +++ b/routers/recorder.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Recorder"]) + @router.post("/v1/record/start", tags=["Recorder"], summary="Start recording browser actions") async def start_recording(session_id: str = Body(...)) -> dict[str, Any]: recorder.start(session_id) diff --git a/routers/referrals.py b/routers/referrals.py index 9a8b1e5..ae13f40 100644 --- a/routers/referrals.py +++ b/routers/referrals.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Referrals"]) + @router.get( "/v1/referrals/catalog", tags=["Referrals"], diff --git a/routers/reports.py b/routers/reports.py index 0b0345c..0ed42fc 100644 --- a/routers/reports.py +++ b/routers/reports.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Reports"]) + @router.post( "/v1/reports/generate", tags=["Reports"], diff --git a/routers/review.py b/routers/review.py index aa58d1d..d4e3251 100644 --- a/routers/review.py +++ b/routers/review.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Review"]) + @router.post("/v1/review/submit", tags=["Review"], summary="Submit extracted data for human review") async def review_submit( data: dict[str, Any] = Body(...), diff --git a/routers/scraping.py b/routers/scraping.py index 0f9d052..36b6909 100644 --- a/routers/scraping.py +++ b/routers/scraping.py @@ -264,7 +264,9 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]: { "max_pages": request.maxPages, "max_depth": request.maxDepth, - "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, + "timeout": request.scrapeOptions.get("timeout", 60) + if request.scrapeOptions + else 60, }, ) return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} diff --git a/routers/scraping_api.py b/routers/scraping_api.py index 87bfacc..bae80fc 100644 --- a/routers/scraping_api.py +++ b/routers/scraping_api.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Scraping"]) + @router.post( "/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system" ) diff --git a/routers/seo.py b/routers/seo.py index f352581..b2147ec 100644 --- a/routers/seo.py +++ b/routers/seo.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["SEO"]) + @router.post("/v1/seo/analyze", tags=["SEO"], summary="Analyze SEO elements from a URL") async def seo_analyze(url: str = Body(...)) -> dict[str, Any]: """Analyze all SEO elements from a URL. diff --git a/routers/sessions.py b/routers/sessions.py index f1f727f..198709c 100644 --- a/routers/sessions.py +++ b/routers/sessions.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Sessions"]) + @router.post("/v1/session/create", tags=["Sessions"], summary="Create a persistent browser session") async def create_session(url: str = Body(...), persist: bool = True) -> dict[str, Any]: """Create a persistent browser session.""" diff --git a/routers/share.py b/routers/share.py index 39693c2..de8e732 100644 --- a/routers/share.py +++ b/routers/share.py @@ -23,6 +23,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Share"]) + @router.post("/v1/share", tags=["Share"], summary="Share scraped content via link") async def share_scrape(data: dict[str, Any] = Body(...)) -> dict[str, Any]: url = data.get("url", "") diff --git a/routers/stats.py b/routers/stats.py index f989d62..a36ffc9 100644 --- a/routers/stats.py +++ b/routers/stats.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Stats"]) + @router.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats") async def stats() -> dict[str, Any]: return { diff --git a/routers/structure.py b/routers/structure.py index 6be264f..6b4e786 100644 --- a/routers/structure.py +++ b/routers/structure.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Structure"]) + @router.post( "/v1/structure/check", tags=["Structure"], diff --git a/routers/system.py b/routers/system.py index 781035d..8cef076 100644 --- a/routers/system.py +++ b/routers/system.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["System"]) + @router.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False) async def metrics() -> Response: """Expose Prometheus metrics for scraping.""" diff --git a/routers/training.py b/routers/training.py index 38b829d..0a22173 100644 --- a/routers/training.py +++ b/routers/training.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Training"]) + @router.post( "/v1/training/classify-license", tags=["Training"], @@ -36,7 +37,9 @@ async def classify_license_endpoint(text: str = Body(...)) -> dict[str, Any]: return {"success": True, "data": result} -@router.post("/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content") +@router.post( + "/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content" +) async def clean_content( text: str = Body(...), strip_names: bool = Body(False), diff --git a/routers/transform.py b/routers/transform.py index d9d72be..1ad280e 100644 --- a/routers/transform.py +++ b/routers/transform.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Transform"]) + @router.post("/v1/transform", tags=["Transform"], summary="Transform data to multiple formats") async def transform_data( data: list[dict[str, Any]] = Body(...), format: str = "csv" diff --git a/routers/untagged.py b/routers/untagged.py index 0a74ed3..cd30fdb 100644 --- a/routers/untagged.py +++ b/routers/untagged.py @@ -18,6 +18,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Untagged"]) + @router.websocket("/v1/stream/{job_id}") async def stream_endpoint(websocket: WebSocket, job_id: str = "live") -> None: await websocket.accept() diff --git a/routers/vision.py b/routers/vision.py index 535c44d..10c1972 100644 --- a/routers/vision.py +++ b/routers/vision.py @@ -26,6 +26,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Vision"]) + @router.post("/v1/vision", tags=["Vision"], summary="Analyze an image with a free vision model") async def vision( question: str = Body("Describe what is visible in this image.", embed=True), diff --git a/routers/webhooks.py b/routers/webhooks.py index 0b8acae..ea46e83 100644 --- a/routers/webhooks.py +++ b/routers/webhooks.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["Webhooks"]) + @router.post( "/v1/webhooks/test", tags=["Webhooks"], diff --git a/routers/x402.py b/routers/x402.py index 56cf02e..3ac180b 100644 --- a/routers/x402.py +++ b/routers/x402.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) router = APIRouter(tags=["x402"]) + @router.get("/v1/x402/pricing", tags=["x402"], summary="Get x402 pricing for all paid operations") async def x402_pricing() -> dict[str, Any]: """Get the price list for pay-per-scrape operations.""" diff --git a/settings.py b/settings.py index 53f0a50..cf8690b 100644 --- a/settings.py +++ b/settings.py @@ -236,7 +236,9 @@ class PrySettings(BaseSettings): proxies.append(f"socks5://{self.tor_socks5_host}:{self.tor_socks5_port}") if self.proxy_enabled and self.proxy_url: if self.proxy_username: - netloc = self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url + netloc = ( + self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url + ) proxies.append( f"{self.proxy_type}://{self.proxy_username}:{self.proxy_password}@{netloc}" ) diff --git a/tests/test_fallback_integration.py b/tests/test_fallback_integration.py index b688979..d2abe2b 100644 --- a/tests/test_fallback_integration.py +++ b/tests/test_fallback_integration.py @@ -17,8 +17,8 @@ from retry import Jitter, RetryConfig, async_retry def _valid_html(text: str = "content") -> str: body = f"

{text}

" * 40 return ( - "T" - "" + 'T' + '' f"{body}" ) @@ -152,7 +152,9 @@ class TestCircuitBreakerIntegration: monkeypatch.setattr(resilience.time, "time", lambda: future) fn = AsyncMock(return_value="ok") - cfg = RetryConfig(max_retries=1, circuit_breaker="ollama_int", base_delay=0.01, jitter=Jitter.NONE) + cfg = RetryConfig( + max_retries=1, circuit_breaker="ollama_int", base_delay=0.01, jitter=Jitter.NONE + ) result = await async_retry(fn, config=cfg) assert result == "ok" @@ -202,7 +204,9 @@ class TestDomainTierMemoryWithRetry: async def test_domain_memory_used_on_retry(self, fresh_registry, scraper_with_mocks): """After domain learns best tier, prefers it.""" s, _mocks = scraper_with_mocks - fresh_registry.record_domain_tier_result("example.com", "archive", success=True, latency=0.1) + fresh_registry.record_domain_tier_result( + "example.com", "archive", success=True, latency=0.1 + ) html = _valid_html("archive content") _mocks["_tier_archive"].return_value = html _mocks["_tier_direct"].return_value = _valid_html("direct") @@ -242,7 +246,9 @@ class TestGracefulDegradation: raise RuntimeError(result.get("error", "scrape failed")) return result - cfg = RetryConfig(max_retries=1, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE) + cfg = RetryConfig( + max_retries=1, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE + ) with pytest.raises(RuntimeError): await async_retry(scrape_and_raise, "https://example.com/page", config=cfg) @@ -255,11 +261,13 @@ class TestRetrySettings: def test_retry_config_from_settings(self) -> None: from settings import PrySettings + settings = PrySettings() assert settings.retry_backoff == "exponential" def test_api_dict_includes_backoff(self) -> None: from settings import PrySettings + settings = PrySettings() api_dict = settings.to_api_dict() assert "retry" in api_dict diff --git a/tests/test_resilience.py b/tests/test_resilience.py index b6606da..b5fee63 100644 --- a/tests/test_resilience.py +++ b/tests/test_resilience.py @@ -16,8 +16,8 @@ from resilience import CircuitBreaker, CircuitState, DomainTierMemory, Resilienc def _valid_html(text: str) -> str: body = f"

{text}

" * 40 return ( - "T" - "" + 'T' + '' f"{body}" ) @@ -159,10 +159,7 @@ class TestUltimateScraperResilience: fresh_registry.record_service_result("flaresolverr", False) html = _valid_html("archive content") - mocks = { - name: AsyncMock(return_value=None) - for name in _TIER_NAMES - } + mocks = {name: AsyncMock(return_value=None) for name in _TIER_NAMES} mocks["_tier_archive"].return_value = html with ExitStack() as stack: for name in _TIER_NAMES: @@ -201,9 +198,7 @@ class TestUltimateScraperResilience: for name in _TIER_NAMES: stack.enter_context(patch.object(scraper, name, new=mocks[name])) await scraper.scrape("https://example.com/page") - assert fresh_registry.best_tier_for_domain( - "example.com", ["direct", "archive"] - ) == "direct" + assert fresh_registry.best_tier_for_domain("example.com", ["direct", "archive"]) == "direct" @pytest.mark.anyio async def test_records_tier_failure_no_success_memory(self, fresh_registry): @@ -214,6 +209,4 @@ class TestUltimateScraperResilience: stack.enter_context(patch.object(scraper, name, new=mocks[name])) result = await scraper.scrape("https://example.com/page") assert result["status"] == "error" - assert fresh_registry.best_tier_for_domain( - "example.com", ["direct", "archive"] - ) is None + assert fresh_registry.best_tier_for_domain("example.com", ["direct", "archive"]) is None diff --git a/tests/test_ultimate_scraper.py b/tests/test_ultimate_scraper.py index 41be2c2..0072a59 100644 --- a/tests/test_ultimate_scraper.py +++ b/tests/test_ultimate_scraper.py @@ -30,12 +30,21 @@ def _clear_registry() -> None: _TIER_ALL = [ - "_tier_direct", "_tier_tls_fingerprint", "_tier_cloudscraper", - "_tier_flaresolverr", "_tier_undetected", "_tier_camoufox", - "_tier_playwright", "_tier_googlebot", "_tier_premium_proxy", - "_tier_archive", "_tier_google_cache", "_tier_tor", + "_tier_direct", + "_tier_tls_fingerprint", + "_tier_cloudscraper", + "_tier_flaresolverr", + "_tier_undetected", + "_tier_camoufox", + "_tier_playwright", + "_tier_googlebot", + "_tier_premium_proxy", + "_tier_archive", + "_tier_google_cache", + "_tier_tor", ] + def test_ultimate_init() -> None: s = UltimateScraper() assert s is not None @@ -132,17 +141,19 @@ async def test_tier_camoufox_failure() -> None: async def test_scrape_falls_through_all_tiers() -> None: _clear_registry() s = UltimateScraper() - with patch.object(s, "_tier_direct", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_tls_fingerprint", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_cloudscraper", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_flaresolverr", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_undetected", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_camoufox", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_playwright", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_googlebot", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_archive", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_google_cache", new=AsyncMock(return_value=None)), \ - patch.object(s, "_tier_tor", new=AsyncMock(return_value=None)): + with ( + patch.object(s, "_tier_direct", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_tls_fingerprint", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_cloudscraper", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_flaresolverr", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_undetected", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_camoufox", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_playwright", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_googlebot", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_archive", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_google_cache", new=AsyncMock(return_value=None)), + patch.object(s, "_tier_tor", new=AsyncMock(return_value=None)), + ): result = await s.scrape("http://example.com") assert result["status"] == "error" diff --git a/url_guard.py b/url_guard.py index 3b12588..b0cb475 100644 --- a/url_guard.py +++ b/url_guard.py @@ -25,15 +25,15 @@ logger = logging.getLogger(__name__) # Private/reserved IP ranges (RFC 1918, RFC 6598, RFC 4193, link-local, loopback) _PRIVATE_NETS: list[str] = [ - "127.0.0.0/8", # loopback - "10.0.0.0/8", # RFC 1918 - "172.16.0.0/12", # RFC 1918 - "192.168.0.0/16", # RFC 1918 - "100.64.0.0/10", # RFC 6598 (CGNAT) - "169.254.0.0/16", # link-local - "fc00::/7", # RFC 4193 (ULA) - "fe80::/10", # link-local IPv6 - "::1/128", # IPv6 loopback + "127.0.0.0/8", # loopback + "10.0.0.0/8", # RFC 1918 + "172.16.0.0/12", # RFC 1918 + "192.168.0.0/16", # RFC 1918 + "100.64.0.0/10", # RFC 6598 (CGNAT) + "169.254.0.0/16", # link-local + "fc00::/7", # RFC 4193 (ULA) + "fe80::/10", # link-local IPv6 + "::1/128", # IPv6 loopback ] # Common internal hostnames that suggest SSRF targeting