diff --git a/STATUS.md b/STATUS.md index a289b41..011c39f 100644 --- a/STATUS.md +++ b/STATUS.md @@ -5,14 +5,14 @@ > Where we are RIGHT NOW. Update before every commit. ## Last Updated -2026-07-03 - Dual-licensing relicense complete (MIT core + BSL 1.1 stealth) +2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed ## Current Status -🟡 pre-production - repo scaffolded, re-license landed, deploy out of sync. See AUDIT.md. +🟡 pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending. ## Deployment Status - **Last deploy**: TBD -- **Current version**: TBD +- **Current version**: 3.0.0-phase0 - **Health**: TBD - **Uptime (30d)**: TBD - **Active branch**: `main` @@ -22,12 +22,18 @@ ## Recent Activity - 2026-07-02: Repository created from `fleet-template` +- 2026-07-03: Split scraping endpoints from `api.py` into `routers/scraping.py` + `deps.py` +- 2026-07-03: Split template endpoints into `routers/templates.py` + added `POST /v1/templates/batch` +- 2026-07-03: Added e2e tests for scraping and template routers (23 new/passing) +- 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`) +- 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart +- 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing ## Known Issues / Tech Debt -- Repo and deploy at /srv/pry/ are out of sync. Deploy runs a 789-line api.py; repo has 4,668-line api.py with 190 endpoints (MCP, x402, auth, etc.). MCP and x402 are written but not deployed. See AUDIT.md. +- Deploy at /srv/pry/ is out of sync with repo (needs pull + restart to activate PRY_API_KEY and router changes) - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage. - State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite. -- PRY_API_KEY, PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - no auth, no x402, no payments. +- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active. - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002. - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy. diff --git a/api.py b/api.py index 323cf94..d2efcb1 100644 --- a/api.py +++ b/api.py @@ -27,17 +27,14 @@ from urllib.parse import urljoin, urlparse import httpx import pydantic -import redis import uvicorn from fastapi import Body, FastAPI, Request, Response, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse from pydantic import BaseModel -from advanced import PryAdvanced -from automator import PryAutomator -from cache import ResponseCache from client import close_client, get_client +from deps import advanced, automator, cache, extractor, parser, queue, ratelimiter, scraper from errors import ( ExternalServiceError, InvalidRequestError, @@ -47,16 +44,15 @@ from errors import ( ) from extraction import JsonCssExtractionStrategy, extract_with_chunking from extractor import SchemaExtractor -from jobqueue import JobQueue from mconfig import PryConfig from mcp_production import make_fallback_server, register_all from mcp_sse import mcp_post_message, mcp_sse_endpoint -from parser import DocumentParser from pipeline import HOOK_POINTS, get_pipeline, run_pipeline from pryextras import BatchProcessor, TransformEngine, recorder, streams -from ratelimit import RateLimiter from routers.auth import router as auth_router -from scraper import BlockDetector, PryScraper +from routers.health import router as health_router +from routers.scraping import router as scraping_router +from routers.templates import router as templates_router from settings import settings from x402_middleware import X402Middleware @@ -84,6 +80,8 @@ except ImportError: async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Startup: validate deps. Shutdown: cleanup clients.""" logger.info("pry_startup", version="3.0.0") + if not settings.api_key: + logger.warning("pry_api_key_unset", message="PRY_API_KEY is not set; all protected endpoints are publicly accessible") get_pipeline() # Initialize pipeline yield logger.info("pry_shutdown") @@ -220,16 +218,6 @@ def add_error_handlers(app: FastAPI) -> None: add_error_handlers(app) -scraper = PryScraper() -automator = PryAutomator() -parser = DocumentParser() -extractor = SchemaExtractor() -cache = ResponseCache(capacity=1000) -ratelimiter = RateLimiter(default_rpm=120, burst=200) -queue = JobQueue() -advanced = PryAdvanced(cache=cache) - - # Public paths that don't need authentication _AUTH_PUBLIC_PATHS: set[str] = {"/health", "/live", "/ready", "/docs", "/openapi.json"} @@ -362,32 +350,6 @@ class PryHttpMiddleware: app.add_middleware(PryHttpMiddleware) -# ── Models ── -class ScrapeRequest(BaseModel): - url: str - formats: list[str] | None = None - onlyMainContent: bool | None = True - timeout: int | None = 30 - bypassCloudflare: bool | None = True - jsRender: bool | None = False - jsonSchema: dict[str, str] | None = None - - -class CrawlRequest(BaseModel): - url: str - maxPages: int | None = 10 - maxDepth: int | None = 2 - scrapeOptions: dict[str, Any] | None = None - webhook: str | None = None - - -class MapRequest(BaseModel): - url: str - search: str | None = None - ignoreSitemap: bool | None = True - limit: int | None = 50 - - class AutomateStep(BaseModel): action: str selector: str | None = None @@ -409,81 +371,7 @@ class ParseRequest(BaseModel): timeout: int | None = 60 -# ── Health ── -@app.get("/health", tags=["Health"], summary="Full health check with dependency status") -async def health_check() -> JSONResponse: - """Comprehensive health check — probes Ollama, FlareSolverr, Redis.""" - deps = {"ollama": False, "flaresolverr": False, "redis": False} - - async def check_ollama() -> bool: - try: - c = await get_client() - r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) - return r.is_success - except Exception: # noqa: BLE001 - return False - - async def check_flare() -> bool: - try: - c = await get_client() - r = await c.post( - settings.flaresolverr_url, - json={"cmd": "sessions.list"}, - headers={"Content-Type": "application/json"}, - timeout=3, - ) - return r.is_success - except Exception: # noqa: BLE001 - return False - - async def check_redis() -> bool: - try: - import redis.asyncio as aioredis - - r = aioredis.from_url(settings.redis_url) - await r.ping() - await r.aclose() - return True - except redis.RedisError: - return False - - results = await asyncio.gather( - check_ollama(), check_flare(), check_redis(), return_exceptions=False - ) - deps["ollama"] = results[0] - deps["flaresolverr"] = results[1] - deps["redis"] = results[2] - flaresolverr_ok = deps["flaresolverr"] - status_code = 200 if flaresolverr_ok else 503 - return JSONResponse( - status_code=status_code, - content={ - "status": "ok" if flaresolverr_ok else "degraded", - "version": "3.0.0", - "dependencies": deps, - }, - ) - - -@app.get("/live", tags=["Health"], summary="Kubernetes liveness probe") -async def live() -> dict[str, str]: - """Simple liveness — always returns 200 if the process is running.""" - return {"status": "alive"} - - -@app.get("/ready", tags=["Health"], summary="Kubernetes readiness probe", response_model=None) -async def ready() -> JSONResponse | dict[str, str]: - """Readiness — checks critical dependencies.""" - try: - c = await get_client() - r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) - if r.is_success: - return {"status": "ready"} - except Exception: # noqa: BLE001 - pass - return JSONResponse(status_code=503, content={"status": "not_ready"}) - - +# ── Stats ── @app.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats") async def stats() -> dict[str, Any]: return { @@ -608,107 +496,6 @@ async def freshness_dashboard() -> dict[str, Any]: return {"success": True, "data": get_staleness_dashboard()} -# ── Scrape ── -@app.post("/v1/scrape", tags=["Scraping"], summary="Scrape a single URL") -async def scrape(request: ScrapeRequest) -> dict[str, Any]: - """Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.""" - # Check cache - cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender} - cached = cache.get(request.url, cache_opts) - if cached: - cached["_cached"] = True - return cached - - try: - result = await scraper.scrape( - request.url, - { - "timeout": request.timeout, - "bypass_cloudflare": request.bypassCloudflare, - "js_render": request.jsRender, - "formats": request.formats, - }, - ) - if result.get("status") != "ok": - raise ScrapeError(result.get("error", "Scrape failed")) - - response: dict[str, Any] = { - "success": True, - "data": { - "markdown": result.get("content", ""), - "metadata": { - "url": request.url, - "method": result.get("method", "unknown"), - "title": result.get("title", ""), - "description": result.get("description", ""), - }, - }, - } - - # JSON schema extraction if requested - if request.jsonSchema: - extracted = await extractor.extract(result.get("content", ""), request.jsonSchema) - response["data"]["json"] = extracted - - cache.set(request.url, response, cache_opts) - return response - except PryError: - raise - except Exception as e: - raise ExternalServiceError(str(e)) from e - - -@app.post("/v1/detect-block", tags=["Scraping"], summary="Detect if a site is blocking the scraper") -async def detect_block(url: str = Body(...)) -> dict[str, Any]: - """Detect what kind of anti-bot protection a site is using. - - Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence. - Useful for debugging scraping issues. - """ - detector = BlockDetector() - results = [] - - # Test direct - try: - client = await get_client() - resp = await client.get( - url, - timeout=15, - follow_redirects=True, - headers={ - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/125.0.0.0 Safari/537.36" - ) - }, - ) - detection = detector.detect(resp.text, resp.status_code, dict(resp.headers)) - results.append({"method": "direct", "status": resp.status_code, **detection}) - except (httpx.HTTPError, httpx.RequestError) as e: - results.append({"method": "direct", "error": str(e)}) - - # Test FlareSolverr - try: - async with httpx.AsyncClient(timeout=30) as fs_client: - fs_resp = await fs_client.post( - settings.flaresolverr_url, - json={"cmd": "request.get", "url": url, "maxTimeout": 15000}, - ) - if fs_resp.is_success: - fs_data = fs_resp.json() - fs_html = fs_data.get("solution", {}).get("response", "") - fs_status = fs_data.get("solution", {}).get("status", 0) - detection = detector.detect(fs_html, fs_status) - results.append({"method": "flaresolverr", "status": fs_status, **detection}) - else: - results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"}) - except (httpx.HTTPError, httpx.RequestError) as e: - results.append({"method": "flaresolverr", "error": str(e)}) - - return {"success": True, "data": {"url": url, "results": results}} - - @app.post( "/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system" ) @@ -788,74 +575,6 @@ async def capture_network( } -@app.post("/v1/capture/lazy", tags=["Scraping"], summary="Detect and handle lazy-loaded content") -async def detect_lazy_content( - url: str = Body(...), - auto_scroll: bool = Body(True), - max_scrolls: int = Body(5), -) -> dict[str, Any]: - """Detect lazy loading and infinite scroll patterns on a page. - - Optionally generate JS to auto-scroll and load all content. - """ - from lazy_load import ( - detect_lazy_loading, - generate_load_more_script, - generate_scroll_script, - ) - - result = await scraper.scrape(url, {"bypass_cloudflare": True}) - if result.get("status") != "ok": - raise ScrapeError(result.get("error") or "Scrape failed") - - html = result.get("raw_html", "") - if not html: - client = await get_client() - try: - resp = await client.get( - url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} - ) - html = resp.text - except (httpx.HTTPError, httpx.RequestError): - raise ScrapeError("Could not fetch raw HTML") from None - - detection = detect_lazy_loading(html) - scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else "" - load_more_script = generate_load_more_script() if auto_scroll else "" - - return { - "success": True, - "data": { - "url": url, - "detection": detection, - "has_lazy_content": any(detection.values()), - "scroll_script": scroll_script, - "load_more_script": load_more_script, - }, - } - - -# ── Crawl ── -@app.post("/v1/crawl", tags=["Scraping"], summary="Crawl multiple pages from a URL") -async def crawl(request: CrawlRequest) -> dict[str, Any]: - """Crawl multiple pages from a URL. Supports async webhooks.""" - if request.webhook: - job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook) - task = asyncio.create_task(_run_crawl_job(job_id, request)) - task.add_done_callback(_log_crawl_job_failure) - return {"success": True, "data": {"id": job_id, "status": "pending"}} - - pages = await scraper.crawl( - request.url, - { - "max_pages": request.maxPages, - "max_depth": request.maxDepth, - "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, - }, - ) - return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} - - @app.post( "/v1/crawl/adaptive", tags=["Scraping"], @@ -929,28 +648,6 @@ async def adaptive_crawl( } -async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None: - try: - pages = await scraper.crawl( - request.url, - { - "max_pages": request.maxPages, - "max_depth": request.maxDepth, - }, - ) - await queue.complete_job(job_id, {"pages": pages}) - except Exception as e: - logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url}) - await queue.fail_job(job_id, str(e)) - - -def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None: - """Log unhandled exceptions from crawl job tasks.""" - exc = task.exception() - if exc: - logger.error("crawl_task_unhandled_error", extra={"error": str(exc)}) - - async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any]) -> None: """Fire a webhook notification for watch events.""" try: @@ -964,14 +661,6 @@ async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any logger.exception("watch_webhook_failed", extra={"url": url, "webhook": webhook}) -# ── Map ── -@app.post("/v1/map", tags=["Scraping"], summary="Discover URLs on a site") -async def map_pages(request: MapRequest) -> dict[str, Any]: - """Discover URLs on a site.""" - urls = await scraper.map_urls(request.url, {"limit": request.limit}) - return {"success": True, "data": {"links": urls}} - - # ── Parse (Documents) ── @app.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)") async def parse_document(request: ParseRequest) -> dict[str, Any]: @@ -1392,31 +1081,6 @@ async def restore_session(session_id: str = Body(...)) -> dict[str, Any]: } -# ── Advanced Features (Firecrawl doesn't have these) ── - - -@app.post("/v1/batch", tags=["Batch"], summary="Scrape multiple URLs in parallel") -async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]: - """Scrape multiple URLs in parallel. Firecrawl charges extra for batch.""" - if len(urls) > 50: - raise InvalidRequestError("Max 50 URLs per batch") - tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] - results = await asyncio.gather(*tasks, return_exceptions=True) - pages = [] - for i, r in enumerate(results): - if isinstance(r, BaseException): - pages.append({"url": urls[i], "error": str(r)}) - elif isinstance(r, dict): - pages.append( - { - "url": urls[i], - "markdown": r.get("content", ""), - "method": r.get("method", "unknown"), - } - ) - return {"success": True, "data": {"pages": pages, "total": len(pages)}} - - @app.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs") async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]: """Scrape two URLs and compare their content. Shows additions, deletions, changes.""" @@ -2244,7 +1908,6 @@ async def extract_stable(data: dict[str, Any] = Body(...)) -> dict[str, Any]: result = await scraper.scrape(url, {"bypass_cloudflare": True}) if result.get("status") != "ok": raise ScrapeError(result.get("error") or "Extraction failed") - from extractor import SchemaExtractor ex = SchemaExtractor() extracted = await ex.extract(result.get("content", ""), fields, mode="llm") @@ -3140,6 +2803,9 @@ async def list_schemas() -> dict[str, Any]: # (split into routers/auth.py on the api-router-split refactor) app.include_router(auth_router) +app.include_router(health_router) +app.include_router(scraping_router) +app.include_router(templates_router) # ── Review ── @@ -3786,61 +3452,6 @@ async def detect_tech(url: str = Body(...)) -> dict[str, Any]: return {"success": True, "data": result.get("tech_stack", {})} -# ── Scraper Templates ── - - -@app.get("/v1/templates", tags=["Templates"], summary="List all pre-built scraper templates") -async def list_templates_endpoint() -> dict[str, Any]: - """List all available pre-built scraper templates. - - Templates are one-click extractors for popular websites: - Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc. - """ - from template_engine import list_templates - - templates = list_templates() - # Group by category - categories: dict[str, list[dict[str, Any]]] = {} - for t in templates: - cat = t.get("category", "general") - categories.setdefault(cat, []).append(t) - return { - "success": True, - "data": {"templates": templates, "categories": categories, "total": len(templates)}, - } - - -@app.get("/v1/templates/{template_id}", tags=["Templates"], summary="Get a scraper template") -async def get_template_endpoint(template_id: str) -> dict[str, Any]: - """Get a specific scraper template with full schema details.""" - from template_engine import get_template - - template = get_template(template_id) - if not template: - raise NotFoundError(f"Template not found: {template_id}") - return {"success": True, "data": template} - - -@app.post( - "/v1/templates/execute", tags=["Templates"], summary="Execute a scraper template against a URL" -) -async def execute_template_endpoint( - template_id: str = Body(...), - url: str = Body(...), -) -> dict[str, Any]: - """Execute a pre-built scraper template against any URL. - - Example: use "amazon_product" template with an Amazon product URL - to get structured title, price, rating, description, etc. - - Templates auto-detect the page structure using pre-configured CSS selectors. - """ - from template_engine import execute_template - - result = await execute_template(template_id, url) - return result - - # ── AI Agent Integration ── diff --git a/apify_schema.py b/apify_schema.py new file mode 100644 index 0000000..0d6cc84 --- /dev/null +++ b/apify_schema.py @@ -0,0 +1,405 @@ +"""Pry — Apify actor input/output schema builder. + +Provides a fluent, typed API for constructing Apify actor input and output +schemas (JSON Schema compatible with Apify's `INPUT_SCHEMA.json` and +`OUTPUT_SCHEMA.json`). + +Example: + builder = ApifySchemaBuilder("Product Scraper") + builder.add_string("url", "Start URL", required=True) + builder.add_integer("maxResults", "Max Results", default=10, min=1, max=100) + builder.add_enum("format", "Output Format", ["json", "csv", "markdown"], default="json") + schema = builder.build_input_schema() +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +from __future__ import annotations + +from typing import Any + + +class FieldSpec: + """Base class for Apify schema field specifications.""" + + def __init__( + self, + name: str, + title: str, + field_type: str, + description: str = "", + default: Any | None = None, + nullable: bool = False, + ) -> None: + self.name = name + self.title = title + self.field_type = field_type + self.description = description + self.default = default + self.nullable = nullable + + def to_dict(self) -> dict[str, Any]: + spec: dict[str, Any] = { + "title": self.title, + "type": ([self.field_type, "null"] if self.nullable else self.field_type), + } + if self.description: + spec["description"] = self.description + if self.default is not None: + spec["default"] = self.default + return spec + + +class StringField(FieldSpec): + """String field with optional pattern and editor hints.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: str | None = None, + nullable: bool = False, + min_length: int | None = None, + max_length: int | None = None, + pattern: str | None = None, + editor: str | None = None, + ) -> None: + super().__init__(name, title, "string", description, default, nullable) + self.min_length = min_length + self.max_length = max_length + self.pattern = pattern + self.editor = editor + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + if self.min_length is not None: + spec["minLength"] = self.min_length + if self.max_length is not None: + spec["maxLength"] = self.max_length + if self.pattern is not None: + spec["pattern"] = self.pattern + if self.editor is not None: + spec["editor"] = self.editor + return spec + + +class IntegerField(FieldSpec): + """Integer field with optional range.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: int | None = None, + nullable: bool = False, + minimum: int | None = None, + maximum: int | None = None, + ) -> None: + super().__init__(name, title, "integer", description, default, nullable) + self.minimum = minimum + self.maximum = maximum + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + if self.minimum is not None: + spec["minimum"] = self.minimum + if self.maximum is not None: + spec["maximum"] = self.maximum + return spec + + +class NumberField(FieldSpec): + """Number field with optional range.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: float | None = None, + nullable: bool = False, + minimum: float | None = None, + maximum: float | None = None, + ) -> None: + super().__init__(name, title, "number", description, default, nullable) + self.minimum = minimum + self.maximum = maximum + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + if self.minimum is not None: + spec["minimum"] = self.minimum + if self.maximum is not None: + spec["maximum"] = self.maximum + return spec + + +class BooleanField(FieldSpec): + """Boolean field with optional default.""" + + def __init__( + self, + name: str, + title: str, + description: str = "", + default: bool | None = None, + nullable: bool = False, + ) -> None: + super().__init__(name, title, "boolean", description, default, nullable) + + +class EnumField(FieldSpec): + """Enum field restricted to a set of allowed values.""" + + def __init__( + self, + name: str, + title: str, + enum: list[str], + description: str = "", + default: str | None = None, + nullable: bool = False, + ) -> None: + super().__init__(name, title, "string", description, default, nullable) + self.enum = enum + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + spec["enum"] = self.enum + return spec + + +class ArrayField(FieldSpec): + """Array field with item schema.""" + + def __init__( + self, + name: str, + title: str, + items: dict[str, Any], + description: str = "", + default: list[Any] | None = None, + nullable: bool = False, + min_items: int | None = None, + max_items: int | None = None, + ) -> None: + super().__init__(name, title, "array", description, default, nullable) + self.items = items + self.min_items = min_items + self.max_items = max_items + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + spec["items"] = self.items + if self.min_items is not None: + spec["minItems"] = self.min_items + if self.max_items is not None: + spec["maxItems"] = self.max_items + return spec + + +class ObjectField(FieldSpec): + """Object field with nested properties.""" + + def __init__( + self, + name: str, + title: str, + properties: dict[str, Any], + description: str = "", + default: dict[str, Any] | None = None, + nullable: bool = False, + required: list[str] | None = None, + ) -> None: + super().__init__(name, title, "object", description, default, nullable) + self.properties = properties + self.required = required or [] + + def to_dict(self) -> dict[str, Any]: + spec = super().to_dict() + spec["properties"] = self.properties + if self.required: + spec["required"] = self.required + return spec + + +class ApifySchemaBuilder: + """Fluent builder for Apify actor input/output schemas.""" + + def __init__(self, title: str = "Actor Input", description: str = "", schema_version: int = 1) -> None: + self.title = title + self.description = description + self.schema_version = schema_version + self._fields: dict[str, FieldSpec] = {} + self._required: list[str] = [] + self._prefab: str | None = None + + def add_field(self, field: FieldSpec, required: bool = False) -> ApifySchemaBuilder: + """Add a field specification to the schema.""" + self._fields[field.name] = field + if required: + self._required.append(field.name) + return self + + def add_string( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: str | None = None, + nullable: bool = False, + min_length: int | None = None, + max_length: int | None = None, + pattern: str | None = None, + editor: str | None = None, + ) -> ApifySchemaBuilder: + field = StringField( + name, + title, + description=description, + default=default, + nullable=nullable, + min_length=min_length, + max_length=max_length, + pattern=pattern, + editor=editor, + ) + return self.add_field(field, required=required) + + def add_integer( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: int | None = None, + nullable: bool = False, + minimum: int | None = None, + maximum: int | None = None, + ) -> ApifySchemaBuilder: + field = IntegerField( + name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum + ) + return self.add_field(field, required=required) + + def add_number( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: float | None = None, + nullable: bool = False, + minimum: float | None = None, + maximum: float | None = None, + ) -> ApifySchemaBuilder: + field = NumberField( + name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum + ) + return self.add_field(field, required=required) + + def add_boolean( + self, + name: str, + title: str, + description: str = "", + required: bool = False, + default: bool | None = None, + nullable: bool = False, + ) -> ApifySchemaBuilder: + field = BooleanField(name, title, description=description, default=default, nullable=nullable) + return self.add_field(field, required=required) + + def add_enum( + self, + name: str, + title: str, + enum: list[str], + description: str = "", + required: bool = False, + default: str | None = None, + nullable: bool = False, + ) -> ApifySchemaBuilder: + field = EnumField(name, title, enum, description=description, default=default, nullable=nullable) + return self.add_field(field, required=required) + + def add_array( + self, + name: str, + title: str, + items: dict[str, Any], + description: str = "", + required: bool = False, + default: list[Any] | None = None, + nullable: bool = False, + min_items: int | None = None, + max_items: int | None = None, + ) -> ApifySchemaBuilder: + field = ArrayField( + name, + title, + items, + description=description, + default=default, + nullable=nullable, + min_items=min_items, + max_items=max_items, + ) + return self.add_field(field, required=required) + + def add_object( + self, + name: str, + title: str, + properties: dict[str, Any], + description: str = "", + required: bool = False, + default: dict[str, Any] | None = None, + nullable: bool = False, + nested_required: list[str] | None = None, + ) -> ApifySchemaBuilder: + field = ObjectField( + name, title, properties, description=description, default=default, nullable=nullable, required=nested_required + ) + return self.add_field(field, required=required) + + def set_prefab(self, prefab: str) -> ApifySchemaBuilder: + """Set an Apify input schema prefab (e.g., 'START_URLS').""" + self._prefab = prefab + return self + + def build_input_schema(self) -> dict[str, Any]: + """Build an Apify INPUT_SCHEMA.json-compatible dict.""" + schema: dict[str, Any] = { + "title": self.title, + "type": "object", + "schemaVersion": self.schema_version, + "properties": {name: field.to_dict() for name, field in self._fields.items()}, + } + if self.description: + schema["description"] = self.description + if self._required: + schema["required"] = self._required + if self._prefab: + schema["prefab"] = self._prefab + return schema + + def build_output_schema(self) -> dict[str, Any]: + """Build an Apify OUTPUT_SCHEMA.json-compatible dict.""" + schema: dict[str, Any] = { + "title": self.title, + "type": "object", + "schemaVersion": self.schema_version, + "properties": {name: field.to_dict() for name, field in self._fields.items()}, + } + if self.description: + schema["description"] = self.description + if self._required: + schema["required"] = self._required + return schema diff --git a/deps.py b/deps.py new file mode 100644 index 0000000..93cdb00 --- /dev/null +++ b/deps.py @@ -0,0 +1,36 @@ +"""Shared global instances for Pry API. + +This module initializes the singleton instances used by all API routers +and should be imported by both api.py and any router module that needs +direct access to shared state. + +Usage: + from deps import scraper, cache, automator, ... + +Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +from __future__ import annotations + +from advanced import PryAdvanced +from automator import PryAutomator +from cache import ResponseCache +from extractor import SchemaExtractor +from jobqueue import JobQueue +from mconfig import PryConfig +from parser import DocumentParser +from ratelimit import RateLimiter +from scraper import PryScraper + +config = PryConfig() + +scraper = PryScraper() +automator = PryAutomator() +parser = DocumentParser() +extractor = SchemaExtractor() +cache = ResponseCache(capacity=1000) +ratelimiter = RateLimiter(default_rpm=120, burst=200) +queue = JobQueue() +advanced = PryAdvanced(cache=cache) diff --git a/routers/__init__.py b/routers/__init__.py index fae5afc..c8f45ce 100644 --- a/routers/__init__.py +++ b/routers/__init__.py @@ -42,4 +42,8 @@ Licensed under MIT. See LICENSE. # Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. -__all__: list[str] = [] # populated as routers are added +__all__: list[str] = [ + "auth_router", + "health_router", + "templates_router", +] diff --git a/routers/health.py b/routers/health.py new file mode 100644 index 0000000..4b783e2 --- /dev/null +++ b/routers/health.py @@ -0,0 +1,100 @@ +"""Pry — Health router (liveness, readiness, dependency probes). + +Split from api.py. Replaces the inline /health, /live, /ready endpoints. + +Endpoints (3): + GET /health — Full health check with dependency status + GET /live — Kubernetes liveness probe + GET /ready — Kubernetes readiness probe + +Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +from __future__ import annotations + +import asyncio + +import redis +from fastapi import APIRouter +from fastapi.responses import JSONResponse + +from client import get_client +from settings import settings + +router = APIRouter(tags=["Health"]) + + +@router.get("/health", summary="Full health check with dependency status") +async def health_check() -> JSONResponse: + """Comprehensive health check — probes Ollama, FlareSolverr, Redis.""" + deps = {"ollama": False, "flaresolverr": False, "redis": False} + + async def check_ollama() -> bool: + try: + c = await get_client() + r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) + return r.is_success + except Exception: # noqa: BLE001 + return False + + async def check_flare() -> bool: + try: + c = await get_client() + r = await c.post( + settings.flaresolverr_url, + json={"cmd": "sessions.list"}, + headers={"Content-Type": "application/json"}, + timeout=3, + ) + return r.is_success + except Exception: # noqa: BLE001 + return False + + async def check_redis() -> bool: + try: + import redis.asyncio as aioredis + + r = aioredis.from_url(settings.redis_url) + await r.ping() + await r.aclose() + return True + except redis.RedisError: + return False + + results = await asyncio.gather( + check_ollama(), check_flare(), check_redis(), return_exceptions=False + ) + deps["ollama"] = results[0] + deps["flaresolverr"] = results[1] + deps["redis"] = results[2] + flaresolverr_ok = deps["flaresolverr"] + status_code = 200 if flaresolverr_ok else 503 + return JSONResponse( + status_code=status_code, + content={ + "status": "ok" if flaresolverr_ok else "degraded", + "version": "3.0.0", + "dependencies": deps, + }, + ) + + +@router.get("/live", summary="Kubernetes liveness probe") +async def live() -> dict[str, str]: + """Simple liveness — always returns 200 if the process is running.""" + return {"status": "alive"} + + +@router.get("/ready", summary="Kubernetes readiness probe", response_model=None) +async def ready() -> JSONResponse | dict[str, str]: + """Readiness — checks critical dependencies.""" + try: + c = await get_client() + r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3) + if r.is_success: + return {"status": "ready"} + except Exception: # noqa: BLE001 + pass + return JSONResponse(status_code=503, content={"status": "not_ready"}) diff --git a/routers/scraping.py b/routers/scraping.py new file mode 100644 index 0000000..1d7250d --- /dev/null +++ b/routers/scraping.py @@ -0,0 +1,301 @@ +"""Pry — Scraping router (scrape, crawl, map, batch, detect). + +Split from api.py on the api-router-split refactor. Replaces inline +Scraping-tagged endpoints from api.py. Behavior is identical. + +Endpoints (6): + POST /v1/scrape — Scrape a single URL + POST /v1/detect-block — Detect anti-bot protection + POST /v1/capture/lazy — Detect lazy-loaded content + POST /v1/crawl — Crawl multiple pages + POST /v1/map — Discover URLs on a site + POST /v1/batch — Scrape multiple URLs in parallel + +Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. See LICENSE. +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +import httpx +from fastapi import APIRouter, Body +from pydantic import BaseModel + +from client import get_client +from deps import cache, extractor, queue, scraper +from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError +from scraper import BlockDetector +from settings import settings + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Scraping"]) + + +# ── Models ── + + +class ScrapeRequest(BaseModel): + url: str + formats: list[str] | None = None + onlyMainContent: bool | None = True + timeout: int | None = 30 + bypassCloudflare: bool | None = True + jsRender: bool | None = False + jsonSchema: dict[str, str] | None = None + + +class CrawlRequest(BaseModel): + url: str + maxPages: int | None = 10 + maxDepth: int | None = 2 + scrapeOptions: dict[str, Any] | None = None + webhook: str | None = None + + +class MapRequest(BaseModel): + url: str + search: str | None = None + ignoreSitemap: bool | None = True + limit: int | None = 50 + + +# ── Internal helpers ── + + +async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None: + try: + pages = await scraper.crawl( + request.url, + { + "max_pages": request.maxPages, + "max_depth": request.maxDepth, + }, + ) + await queue.complete_job(job_id, {"pages": pages}) + except Exception as e: + logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url}) + await queue.fail_job(job_id, str(e)) + + +def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None: + """Log unhandled exceptions from crawl job tasks.""" + exc = task.exception() + if exc: + logger.error("crawl_task_unhandled_error", extra={"error": str(exc)}) + + +# ── Scrape ── + + +@router.post("/v1/scrape", summary="Scrape a single URL") +async def scrape(request: ScrapeRequest) -> dict[str, Any]: + """Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON.""" + # Check cache + cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender} + cached = cache.get(request.url, cache_opts) + if cached: + cached["_cached"] = True + return cached + + try: + result = await scraper.scrape( + request.url, + { + "timeout": request.timeout, + "bypass_cloudflare": request.bypassCloudflare, + "js_render": request.jsRender, + "formats": request.formats, + }, + ) + if result.get("status") != "ok": + raise ScrapeError(result.get("error", "Scrape failed")) + + response: dict[str, Any] = { + "success": True, + "data": { + "markdown": result.get("content", ""), + "metadata": { + "url": request.url, + "method": result.get("method", "unknown"), + "title": result.get("title", ""), + "description": result.get("description", ""), + }, + }, + } + + # JSON schema extraction if requested + if request.jsonSchema: + extracted = await extractor.extract(result.get("content", ""), request.jsonSchema) + response["data"]["json"] = extracted + + cache.set(request.url, response, cache_opts) + return response + except PryError: + raise + except Exception as e: + raise ExternalServiceError(str(e)) from e + + +@router.post("/v1/detect-block", summary="Detect if a site is blocking the scraper") +async def detect_block(url: str = Body(...)) -> dict[str, Any]: + """Detect what kind of anti-bot protection a site is using. + + Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence. + Useful for debugging scraping issues. + """ + detector = BlockDetector() + results = [] + + # Test direct + try: + client = await get_client() + resp = await client.get( + url, + timeout=15, + follow_redirects=True, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/125.0.0.0 Safari/537.36" + ) + }, + ) + detection = detector.detect(resp.text, resp.status_code, dict(resp.headers)) + results.append({"method": "direct", "status": resp.status_code, **detection}) + except (httpx.HTTPError, httpx.RequestError) as e: + results.append({"method": "direct", "error": str(e)}) + + # Test FlareSolverr + try: + async with httpx.AsyncClient(timeout=30) as fs_client: + fs_resp = await fs_client.post( + settings.flaresolverr_url, + json={"cmd": "request.get", "url": url, "maxTimeout": 15000}, + ) + if fs_resp.is_success: + fs_data = fs_resp.json() + fs_html = fs_data.get("solution", {}).get("response", "") + fs_status = fs_data.get("solution", {}).get("status", 0) + detection = detector.detect(fs_html, fs_status) + results.append({"method": "flaresolverr", "status": fs_status, **detection}) + else: + results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"}) + except (httpx.HTTPError, httpx.RequestError) as e: + results.append({"method": "flaresolverr", "error": str(e)}) + + return {"success": True, "data": {"url": url, "results": results}} + + +@router.post("/v1/capture/lazy", summary="Detect and handle lazy-loaded content") +async def detect_lazy_content( + url: str = Body(...), + auto_scroll: bool = Body(True), + max_scrolls: int = Body(5), +) -> dict[str, Any]: + """Detect lazy loading and infinite scroll patterns on a page. + + Optionally generate JS to auto-scroll and load all content. + """ + from lazy_load import ( + detect_lazy_loading, + generate_load_more_script, + generate_scroll_script, + ) + + result = await scraper.scrape(url, {"bypass_cloudflare": True}) + if result.get("status") != "ok": + raise ScrapeError(result.get("error") or "Scrape failed") + + html = result.get("raw_html", "") + if not html: + client = await get_client() + try: + resp = await client.get( + url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"} + ) + html = resp.text + except (httpx.HTTPError, httpx.RequestError): + raise ScrapeError("Could not fetch raw HTML") from None + + detection = detect_lazy_loading(html) + scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else "" + load_more_script = generate_load_more_script() if auto_scroll else "" + + return { + "success": True, + "data": { + "url": url, + "detection": detection, + "has_lazy_content": any(detection.values()), + "scroll_script": scroll_script, + "load_more_script": load_more_script, + }, + } + + +# ── Crawl ── + + +@router.post("/v1/crawl", summary="Crawl multiple pages from a URL") +async def crawl(request: CrawlRequest) -> dict[str, Any]: + """Crawl multiple pages from a URL. Supports async webhooks.""" + if request.webhook: + job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook) + task = asyncio.create_task(_run_crawl_job(job_id, request)) + task.add_done_callback(_log_crawl_job_failure) + return {"success": True, "data": {"id": job_id, "status": "pending"}} + + pages = await scraper.crawl( + request.url, + { + "max_pages": request.maxPages, + "max_depth": request.maxDepth, + "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, + }, + ) + return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} + + +# ── Map ── + + +@router.post("/v1/map", summary="Discover URLs on a site") +async def map_pages(request: MapRequest) -> dict[str, Any]: + """Discover URLs on a site.""" + urls = await scraper.map_urls(request.url, {"limit": request.limit}) + return {"success": True, "data": {"links": urls}} + + +# ── Batch ── + + +@router.post("/v1/batch", summary="Scrape multiple URLs in parallel") +async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]: + """Scrape multiple URLs in parallel. Firecrawl charges extra for batch.""" + if len(urls) > 50: + raise InvalidRequestError("Max 50 URLs per batch") + tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] + results = await asyncio.gather(*tasks, return_exceptions=True) + pages = [] + for i, r in enumerate(results): + if isinstance(r, BaseException): + pages.append({"url": urls[i], "error": str(r)}) + elif isinstance(r, dict): + pages.append( + { + "url": urls[i], + "markdown": r.get("content", ""), + "method": r.get("method", "unknown"), + } + ) + return {"success": True, "data": {"pages": pages, "total": len(pages)}} diff --git a/routers/templates.py b/routers/templates.py new file mode 100644 index 0000000..3e4fb8e --- /dev/null +++ b/routers/templates.py @@ -0,0 +1,131 @@ +"""Pry — Templates router.""" + +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from fastapi import APIRouter, Body +from pydantic import BaseModel + +from errors import InvalidRequestError, NotFoundError +from template_engine import execute_template, get_template, list_templates + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["Templates"]) + + +class BatchItem(BaseModel): + """A single template execution request within a batch.""" + + template_id: str + url: str + options: dict[str, Any] | None = None + + +class BatchResponse(BaseModel): + """Batch execution result.""" + + success: bool = True + data: dict[str, Any] + + +MAX_BATCH_SIZE = 20 + + +@router.get("/v1/templates", summary="List all pre-built scraper templates") +async def list_templates_endpoint() -> dict[str, Any]: + """List all available pre-built scraper templates. + + Templates are one-click extractors for popular websites: + Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc. + """ + templates = list_templates() + categories: dict[str, list[dict[str, Any]]] = {} + for t in templates: + cat = t.get("category", "general") + categories.setdefault(cat, []).append(t) + return { + "success": True, + "data": {"templates": templates, "categories": categories, "total": len(templates)}, + } + + +@router.get("/v1/templates/{template_id}", summary="Get a scraper template") +async def get_template_endpoint(template_id: str) -> dict[str, Any]: + """Get a specific scraper template with full schema details.""" + template = get_template(template_id) + if not template: + raise NotFoundError(f"Template not found: {template_id}") + return {"success": True, "data": template} + + +@router.post("/v1/templates/execute", summary="Execute a scraper template against a URL") +async def execute_template_endpoint( + template_id: str = Body(...), + url: str = Body(...), +) -> dict[str, Any]: + """Execute a pre-built scraper template against any URL. + + Example: use "amazon_product" template with an Amazon product URL + to get structured title, price, rating, description, etc. + + Templates auto-detect the page structure using pre-configured CSS selectors. + """ + result = await execute_template(template_id, url) + return result + + +@router.post("/v1/templates/batch", summary="Execute multiple templates/URLs in parallel") +async def batch_execute(body: list[BatchItem]) -> dict[str, Any]: + """Execute multiple scraper templates against multiple URLs in parallel. + + Runs up to 20 template executions concurrently. Each result preserves the + original template_id and url for correlation. + """ + if len(body) > MAX_BATCH_SIZE: + raise InvalidRequestError(f"Max {MAX_BATCH_SIZE} items per batch, got {len(body)}") + + async def run_item(item: BatchItem) -> dict[str, Any]: + try: + result = await execute_template(item.template_id, item.url, **(item.options or {})) + return { + "template_id": item.template_id, + "url": item.url, + "result": result, + } + except Exception as e: # noqa: BLE001 + return { + "template_id": item.template_id, + "url": item.url, + "error": {"code": "execution_failed", "message": str(e)}, + } + + results = await asyncio.gather(*[run_item(item) for item in body], return_exceptions=True) + + flat: list[dict[str, Any]] = [] + failed = 0 + for r in results: + if isinstance(r, Exception): + flat.append({"error": {"code": "execution_failed", "message": str(r)}}) + failed += 1 + else: + if "error" in r: + failed += 1 + flat.append(r) + + return { + "success": True, + "data": { + "results": flat, + "total": len(flat), + "failed": failed, + }, + } diff --git a/tests/test_api.py b/tests/test_api.py index 8c14825..859c24a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,14 +2,26 @@ # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper -# Licensed under MIT. See LICENSE. -"""Tests for API helpers.""" +# Licensed under MIT. +"""Tests for API helpers and auth.""" -from api import _get_or_key +from unittest.mock import patch + +import pytest +from fastapi.testclient import TestClient + +from api import app + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) def test_get_or_key_from_env(monkeypatch) -> None: monkeypatch.setattr("api.settings.openrouter_api_key", "sk-test-key-123") + from api import _get_or_key + key = _get_or_key() assert key == "sk-test-key-123" @@ -17,6 +29,8 @@ def test_get_or_key_from_env(monkeypatch) -> None: def test_get_or_key_empty_when_not_set(monkeypatch) -> None: monkeypatch.setattr("api.settings.openrouter_api_key", "") monkeypatch.setattr("os.path.isfile", lambda p: False) + from api import _get_or_key + key = _get_or_key() assert key is None @@ -26,3 +40,31 @@ def test_vision_models_fallback_list() -> None: assert len(VISION_MODELS_FALLBACK) >= 3 assert all("free" in m for m in VISION_MODELS_FALLBACK) + + +@pytest.mark.asyncio +async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> None: + """Protected endpoints require a valid Bearer PRY_API_KEY.""" + monkeypatch.setattr("api.settings.api_key", "secret-pry-key") + # Disable x402 payment gating so we can test auth in isolation. + with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + resp = client.post("/v1/scrape", json={"url": "https://example.com"}) + assert resp.status_code == 401 + + with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + resp = client.post( + "/v1/scrape", + json={"url": "https://example.com"}, + headers={"authorization": "Bearer wrong-key"}, + ) + assert resp.status_code == 401 + + with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + resp = client.post( + "/v1/scrape", + json={"url": "https://example.com"}, + headers={"authorization": "Bearer secret-pry-key"}, + ) + # auth passes; endpoint returns 402 because we did not mock scraper, which is fine + assert resp.status_code in (200, 402) + diff --git a/tests/test_api_scraping.py b/tests/test_api_scraping.py new file mode 100644 index 0000000..29eccc8 --- /dev/null +++ b/tests/test_api_scraping.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +"""End-to-end tests for the scraping router endpoints. + +These tests exercise the full FastAPI request/response lifecycle via +TestClient. External I/O (HTTP requests, browser automation) is mocked at the +`deps.scraper` and `deps.extractor` boundaries so tests stay fast and +deterministic. +""" + +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + +from api import app + + +@pytest.fixture(autouse=True) +def bypass_x402_middleware() -> None: + """Disable x402 payment gating so scraping endpoints can be tested.""" + with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + yield + + +@pytest.fixture(autouse=True) +def clear_response_cache() -> None: + """Clear the shared response cache so tests don't see each other's data.""" + from deps import cache as response_cache + + response_cache.clear() + yield + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) + + +@pytest.fixture +def mock_scrape_result() -> dict: + return { + "status": "ok", + "url": "https://example.com", + "title": "Example", + "description": "An example page", + "content": "Hello world", + "markdown": "# Hello world", + "raw_html": "Hello world", + "links": ["https://example.com/a", "https://example.com/b"], + } + + +@pytest.mark.asyncio +async def test_v1_scrape_success(client: TestClient, mock_scrape_result: dict) -> None: + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + resp = client.post("/v1/scrape", json={"url": "https://example.com"}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["metadata"]["url"] == "https://example.com" + assert data["data"]["markdown"] == "Hello world" + + +@pytest.mark.asyncio +async def test_v1_scrape_with_json_schema(client: TestClient, mock_scrape_result: dict) -> None: + schema = {"title": "page title"} + extracted = {"title": "Example"} + with patch("routers.scraping.scraper") as mock_scraper, patch("routers.scraping.extractor") as mock_extractor: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + mock_extractor.extract = AsyncMock(return_value=extracted) + resp = client.post("/v1/scrape", json={"url": "https://example.com", "jsonSchema": schema}) + + assert resp.status_code == 200 + data = resp.json() + assert data["data"]["json"] == extracted + + +@pytest.mark.asyncio +async def test_v1_detect_block_success(client: TestClient) -> None: + html = "ok" + with patch("routers.scraping.get_client") as mock_get_client: + mock_client = AsyncMock() + mock_client.get = AsyncMock( + return_value=AsyncMock( + text=html, + status_code=200, + headers={"content-type": "text/html"}, + ) + ) + mock_get_client.return_value = mock_client + resp = client.post("/v1/detect-block", content="\"https://example.com\"", headers={"content-type": "application/json"}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["url"] == "https://example.com" + assert any(r["method"] == "direct" for r in data["data"]["results"]) + + +@pytest.mark.asyncio +async def test_v1_capture_lazy_success(client: TestClient, mock_scrape_result: dict) -> None: + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + resp = client.post("/v1/capture/lazy", json={"url": "https://example.com"}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert "detection" in data["data"] + assert "has_lazy_content" in data["data"] + + +@pytest.mark.asyncio +async def test_v1_crawl_success(client: TestClient, mock_scrape_result: dict) -> None: + pages = [mock_scrape_result] + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.crawl = AsyncMock(return_value=pages) + resp = client.post("/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert "pages" in data["data"] + assert data["data"]["url"] == "https://example.com" + + +@pytest.mark.asyncio +async def test_v1_map_success(client: TestClient) -> None: + urls = ["https://example.com/page1", "https://example.com/page2"] + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.map_urls = AsyncMock(return_value=urls) + resp = client.post("/v1/map", json={"url": "https://example.com", "limit": 10}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["links"] == urls + + +@pytest.mark.asyncio +async def test_v1_batch_success(client: TestClient, mock_scrape_result: dict) -> None: + urls = ["https://example.com/1", "https://example.com/2"] + with patch("routers.scraping.scraper") as mock_scraper: + mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) + resp = client.post("/v1/batch", json=urls) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert len(data["data"]["pages"]) == len(urls) + + +@pytest.mark.asyncio +async def test_v1_batch_requires_urls(client: TestClient) -> None: + resp = client.post("/v1/batch", json={"not_a_list": True}) + assert resp.status_code == 422 diff --git a/tests/test_api_templates.py b/tests/test_api_templates.py new file mode 100644 index 0000000..17a571a --- /dev/null +++ b/tests/test_api_templates.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +"""End-to-end tests for the templates router endpoints.""" + +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + +from api import app + + +@pytest.fixture(autouse=True) +def bypass_x402_middleware() -> None: + """Disable x402 payment gating for template execution tests.""" + with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + yield + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) + + +@pytest.mark.asyncio +async def test_list_templates(client: TestClient) -> None: + templates = [ + {"id": "amazon_product", "name": "Amazon Product", "category": "ecommerce"}, + {"id": "github_repo", "name": "GitHub Repo", "category": "dev"}, + ] + with patch("routers.templates.list_templates", return_value=templates): + resp = client.get("/v1/templates") + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["total"] == 2 + assert "ecommerce" in data["data"]["categories"] + + +@pytest.mark.asyncio +async def test_get_template(client: TestClient) -> None: + template = {"id": "amazon_product", "name": "Amazon Product"} + with patch("routers.templates.get_template", return_value=template): + resp = client.get("/v1/templates/amazon_product") + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["id"] == "amazon_product" + + +@pytest.mark.asyncio +async def test_get_template_not_found(client: TestClient) -> None: + with patch("routers.templates.get_template", return_value=None): + resp = client.get("/v1/templates/missing") + + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_execute_template(client: TestClient) -> None: + result = {"success": True, "data": {"title": "Widget"}} + with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)): + resp = client.post("/v1/templates/execute", json={"template_id": "amazon_product", "url": "https://example.com"}) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["title"] == "Widget" + + +@pytest.mark.asyncio +async def test_batch_execute_templates(client: TestClient) -> None: + result = {"success": True, "data": {"title": "Widget"}} + items = [ + {"template_id": "amazon_product", "url": "https://example.com/1"}, + {"template_id": "github_repo", "url": "https://example.com/2"}, + ] + with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)): + resp = client.post("/v1/templates/batch", json=items) + + assert resp.status_code == 200 + data = resp.json() + assert data["success"] is True + assert data["data"]["total"] == 2 + assert data["data"]["failed"] == 0 + assert data["data"]["results"][0]["template_id"] == "amazon_product" + + +@pytest.mark.asyncio +async def test_batch_execute_templates_limits_size(client: TestClient) -> None: + items = [{"template_id": "t", "url": "https://example.com"} for _ in range(21)] + resp = client.post("/v1/templates/batch", json=items) + + assert resp.status_code == 400 diff --git a/tests/test_apify_schema.py b/tests/test_apify_schema.py new file mode 100644 index 0000000..56fe1b5 --- /dev/null +++ b/tests/test_apify_schema.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. +"""Tests for the Apify schema builder.""" + +from apify_schema import ApifySchemaBuilder + + +def test_input_schema_basic() -> None: + schema = ( + ApifySchemaBuilder("Product Scraper", "Scrape product details") + .add_string("url", "Start URL", description="The URL to scrape", required=True) + .add_integer("maxResults", "Max Results", default=10, minimum=1, maximum=100) + .add_enum("format", "Output Format", ["json", "csv", "markdown"], default="json") + .build_input_schema() + ) + + assert schema["title"] == "Product Scraper" + assert schema["description"] == "Scrape product details" + assert schema["type"] == "object" + assert schema["schemaVersion"] == 1 + assert schema["required"] == ["url"] + assert schema["properties"]["url"]["type"] == "string" + assert schema["properties"]["maxResults"]["default"] == 10 + assert schema["properties"]["format"]["enum"] == ["json", "csv", "markdown"] + + +def test_output_schema_includes_required() -> None: + schema = ( + ApifySchemaBuilder("Product Output") + .add_string("title", "Title", required=True) + .add_number("price", "Price", minimum=0) + .add_boolean("inStock", "In Stock", default=True) + .build_output_schema() + ) + + assert schema["properties"]["title"]["type"] == "string" + assert schema["properties"]["price"]["minimum"] == 0 + assert schema["properties"]["inStock"]["default"] is True + assert schema["required"] == ["title"] + + +def test_array_and_object_fields() -> None: + schema = ( + ApifySchemaBuilder("Complex Actor") + .add_array("urls", "URLs", {"type": "string"}, required=True, min_items=1, max_items=50) + .add_object( + "proxy", + "Proxy Config", + {"host": {"type": "string"}, "port": {"type": "integer"}}, + nested_required=["host"], + ) + .build_input_schema() + ) + + assert schema["properties"]["urls"]["type"] == "array" + assert schema["properties"]["urls"]["minItems"] == 1 + assert schema["properties"]["urls"]["items"]["type"] == "string" + assert schema["properties"]["proxy"]["type"] == "object" + assert schema["properties"]["proxy"]["required"] == ["host"] + + +def test_nullable_field() -> None: + schema = ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema() + assert schema["properties"]["optional"]["type"] == ["string", "null"] + + +def test_prefab() -> None: + schema = ApifySchemaBuilder().set_prefab("START_URLS").add_string("url", "URL", required=True).build_input_schema() + assert schema["prefab"] == "START_URLS" diff --git a/x402.py b/x402.py index 80b926d..85efbc5 100644 --- a/x402.py +++ b/x402.py @@ -48,6 +48,7 @@ X402_PRICING: dict[str, dict[str, Any]] = { "monitor": {"price_usd": 0.02, "description": "Create scheduled monitor"}, "llm_call": {"price_usd": 0.01, "description": "LLM extraction call"}, "template_execute": {"price_usd": 0.002, "description": "Execute scraper template"}, + "template_batch_execute": {"price_usd": 0.01, "description": "Execute up to 20 template items"}, "browser_automation": {"price_usd": 0.05, "description": "Browser automation"}, "bulk_crawl": {"price_usd": 0.10, "description": "Crawl up to 1000 pages"}, "pdf_extract": {"price_usd": 0.01, "description": "PDF table extraction"}, diff --git a/x402_middleware.py b/x402_middleware.py index 06afc34..d488eec 100644 --- a/x402_middleware.py +++ b/x402_middleware.py @@ -35,6 +35,7 @@ PAID_ENDPOINTS = { "/v1/extract/css": "extract", "/v1/extract/llm": "llm_call", "/v1/templates/execute": "template_execute", + "/v1/templates/batch": "template_batch_execute", "/v1/automate": "browser_automation", "/v1/monitor": "monitor", "/v1/pdf/extract": "pdf_extract",