Merge remote-tracking branch 'bare/main' into feat/sql-migration-top3
This commit is contained in:
commit
971b817d3d
14 changed files with 1379 additions and 408 deletions
16
STATUS.md
16
STATUS.md
|
|
@ -5,14 +5,14 @@
|
|||
> Where we are RIGHT NOW. Update before every commit.
|
||||
|
||||
## Last Updated
|
||||
2026-07-03 - Dual-licensing relicense complete (MIT core + BSL 1.1 stealth)
|
||||
2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed
|
||||
|
||||
## Current Status
|
||||
🟡 pre-production - repo scaffolded, re-license landed, deploy out of sync. See AUDIT.md.
|
||||
🟡 pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending.
|
||||
|
||||
## Deployment Status
|
||||
- **Last deploy**: TBD
|
||||
- **Current version**: TBD
|
||||
- **Current version**: 3.0.0-phase0
|
||||
- **Health**: TBD
|
||||
- **Uptime (30d)**: TBD
|
||||
- **Active branch**: `main`
|
||||
|
|
@ -22,12 +22,18 @@
|
|||
|
||||
## Recent Activity
|
||||
- 2026-07-02: Repository created from `fleet-template`
|
||||
- 2026-07-03: Split scraping endpoints from `api.py` into `routers/scraping.py` + `deps.py`
|
||||
- 2026-07-03: Split template endpoints into `routers/templates.py` + added `POST /v1/templates/batch`
|
||||
- 2026-07-03: Added e2e tests for scraping and template routers (23 new/passing)
|
||||
- 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`)
|
||||
- 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart
|
||||
- 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing
|
||||
|
||||
## Known Issues / Tech Debt
|
||||
- Repo and deploy at /srv/pry/ are out of sync. Deploy runs a 789-line api.py; repo has 4,668-line api.py with 190 endpoints (MCP, x402, auth, etc.). MCP and x402 are written but not deployed. See AUDIT.md.
|
||||
- Deploy at /srv/pry/ is out of sync with repo (needs pull + restart to activate PRY_API_KEY and router changes)
|
||||
- All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
|
||||
- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite.
|
||||
- PRY_API_KEY, PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - no auth, no x402, no payments.
|
||||
- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
|
||||
- License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
|
||||
- mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.
|
||||
|
||||
|
|
|
|||
409
api.py
409
api.py
|
|
@ -27,17 +27,14 @@ from urllib.parse import urljoin, urlparse
|
|||
|
||||
import httpx
|
||||
import pydantic
|
||||
import redis
|
||||
import uvicorn
|
||||
from fastapi import Body, FastAPI, Request, Response, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from advanced import PryAdvanced
|
||||
from automator import PryAutomator
|
||||
from cache import ResponseCache
|
||||
from client import close_client, get_client
|
||||
from deps import advanced, automator, cache, extractor, parser, queue, ratelimiter, scraper
|
||||
from errors import (
|
||||
ExternalServiceError,
|
||||
InvalidRequestError,
|
||||
|
|
@ -47,16 +44,15 @@ from errors import (
|
|||
)
|
||||
from extraction import JsonCssExtractionStrategy, extract_with_chunking
|
||||
from extractor import SchemaExtractor
|
||||
from jobqueue import JobQueue
|
||||
from mconfig import PryConfig
|
||||
from mcp_production import make_fallback_server, register_all
|
||||
from mcp_sse import mcp_post_message, mcp_sse_endpoint
|
||||
from parser import DocumentParser
|
||||
from pipeline import HOOK_POINTS, get_pipeline, run_pipeline
|
||||
from pryextras import BatchProcessor, TransformEngine, recorder, streams
|
||||
from ratelimit import RateLimiter
|
||||
from routers.auth import router as auth_router
|
||||
from scraper import BlockDetector, PryScraper
|
||||
from routers.health import router as health_router
|
||||
from routers.scraping import router as scraping_router
|
||||
from routers.templates import router as templates_router
|
||||
from settings import settings
|
||||
from x402_middleware import X402Middleware
|
||||
|
||||
|
|
@ -84,6 +80,8 @@ except ImportError:
|
|||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
"""Startup: validate deps. Shutdown: cleanup clients."""
|
||||
logger.info("pry_startup", version="3.0.0")
|
||||
if not settings.api_key:
|
||||
logger.warning("pry_api_key_unset", message="PRY_API_KEY is not set; all protected endpoints are publicly accessible")
|
||||
get_pipeline() # Initialize pipeline
|
||||
yield
|
||||
logger.info("pry_shutdown")
|
||||
|
|
@ -220,16 +218,6 @@ def add_error_handlers(app: FastAPI) -> None:
|
|||
|
||||
add_error_handlers(app)
|
||||
|
||||
scraper = PryScraper()
|
||||
automator = PryAutomator()
|
||||
parser = DocumentParser()
|
||||
extractor = SchemaExtractor()
|
||||
cache = ResponseCache(capacity=1000)
|
||||
ratelimiter = RateLimiter(default_rpm=120, burst=200)
|
||||
queue = JobQueue()
|
||||
advanced = PryAdvanced(cache=cache)
|
||||
|
||||
|
||||
# Public paths that don't need authentication
|
||||
_AUTH_PUBLIC_PATHS: set[str] = {"/health", "/live", "/ready", "/docs", "/openapi.json"}
|
||||
|
||||
|
|
@ -362,32 +350,6 @@ class PryHttpMiddleware:
|
|||
app.add_middleware(PryHttpMiddleware)
|
||||
|
||||
|
||||
# ── Models ──
|
||||
class ScrapeRequest(BaseModel):
|
||||
url: str
|
||||
formats: list[str] | None = None
|
||||
onlyMainContent: bool | None = True
|
||||
timeout: int | None = 30
|
||||
bypassCloudflare: bool | None = True
|
||||
jsRender: bool | None = False
|
||||
jsonSchema: dict[str, str] | None = None
|
||||
|
||||
|
||||
class CrawlRequest(BaseModel):
|
||||
url: str
|
||||
maxPages: int | None = 10
|
||||
maxDepth: int | None = 2
|
||||
scrapeOptions: dict[str, Any] | None = None
|
||||
webhook: str | None = None
|
||||
|
||||
|
||||
class MapRequest(BaseModel):
|
||||
url: str
|
||||
search: str | None = None
|
||||
ignoreSitemap: bool | None = True
|
||||
limit: int | None = 50
|
||||
|
||||
|
||||
class AutomateStep(BaseModel):
|
||||
action: str
|
||||
selector: str | None = None
|
||||
|
|
@ -409,81 +371,7 @@ class ParseRequest(BaseModel):
|
|||
timeout: int | None = 60
|
||||
|
||||
|
||||
# ── Health ──
|
||||
@app.get("/health", tags=["Health"], summary="Full health check with dependency status")
|
||||
async def health_check() -> JSONResponse:
|
||||
"""Comprehensive health check — probes Ollama, FlareSolverr, Redis."""
|
||||
deps = {"ollama": False, "flaresolverr": False, "redis": False}
|
||||
|
||||
async def check_ollama() -> bool:
|
||||
try:
|
||||
c = await get_client()
|
||||
r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3)
|
||||
return r.is_success
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
async def check_flare() -> bool:
|
||||
try:
|
||||
c = await get_client()
|
||||
r = await c.post(
|
||||
settings.flaresolverr_url,
|
||||
json={"cmd": "sessions.list"},
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=3,
|
||||
)
|
||||
return r.is_success
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
async def check_redis() -> bool:
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
r = aioredis.from_url(settings.redis_url)
|
||||
await r.ping()
|
||||
await r.aclose()
|
||||
return True
|
||||
except redis.RedisError:
|
||||
return False
|
||||
|
||||
results = await asyncio.gather(
|
||||
check_ollama(), check_flare(), check_redis(), return_exceptions=False
|
||||
)
|
||||
deps["ollama"] = results[0]
|
||||
deps["flaresolverr"] = results[1]
|
||||
deps["redis"] = results[2]
|
||||
flaresolverr_ok = deps["flaresolverr"]
|
||||
status_code = 200 if flaresolverr_ok else 503
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"status": "ok" if flaresolverr_ok else "degraded",
|
||||
"version": "3.0.0",
|
||||
"dependencies": deps,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/live", tags=["Health"], summary="Kubernetes liveness probe")
|
||||
async def live() -> dict[str, str]:
|
||||
"""Simple liveness — always returns 200 if the process is running."""
|
||||
return {"status": "alive"}
|
||||
|
||||
|
||||
@app.get("/ready", tags=["Health"], summary="Kubernetes readiness probe", response_model=None)
|
||||
async def ready() -> JSONResponse | dict[str, str]:
|
||||
"""Readiness — checks critical dependencies."""
|
||||
try:
|
||||
c = await get_client()
|
||||
r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3)
|
||||
if r.is_success:
|
||||
return {"status": "ready"}
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return JSONResponse(status_code=503, content={"status": "not_ready"})
|
||||
|
||||
|
||||
# ── Stats ──
|
||||
@app.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats")
|
||||
async def stats() -> dict[str, Any]:
|
||||
return {
|
||||
|
|
@ -608,107 +496,6 @@ async def freshness_dashboard() -> dict[str, Any]:
|
|||
return {"success": True, "data": get_staleness_dashboard()}
|
||||
|
||||
|
||||
# ── Scrape ──
|
||||
@app.post("/v1/scrape", tags=["Scraping"], summary="Scrape a single URL")
|
||||
async def scrape(request: ScrapeRequest) -> dict[str, Any]:
|
||||
"""Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON."""
|
||||
# Check cache
|
||||
cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender}
|
||||
cached = cache.get(request.url, cache_opts)
|
||||
if cached:
|
||||
cached["_cached"] = True
|
||||
return cached
|
||||
|
||||
try:
|
||||
result = await scraper.scrape(
|
||||
request.url,
|
||||
{
|
||||
"timeout": request.timeout,
|
||||
"bypass_cloudflare": request.bypassCloudflare,
|
||||
"js_render": request.jsRender,
|
||||
"formats": request.formats,
|
||||
},
|
||||
)
|
||||
if result.get("status") != "ok":
|
||||
raise ScrapeError(result.get("error", "Scrape failed"))
|
||||
|
||||
response: dict[str, Any] = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": result.get("content", ""),
|
||||
"metadata": {
|
||||
"url": request.url,
|
||||
"method": result.get("method", "unknown"),
|
||||
"title": result.get("title", ""),
|
||||
"description": result.get("description", ""),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# JSON schema extraction if requested
|
||||
if request.jsonSchema:
|
||||
extracted = await extractor.extract(result.get("content", ""), request.jsonSchema)
|
||||
response["data"]["json"] = extracted
|
||||
|
||||
cache.set(request.url, response, cache_opts)
|
||||
return response
|
||||
except PryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ExternalServiceError(str(e)) from e
|
||||
|
||||
|
||||
@app.post("/v1/detect-block", tags=["Scraping"], summary="Detect if a site is blocking the scraper")
|
||||
async def detect_block(url: str = Body(...)) -> dict[str, Any]:
|
||||
"""Detect what kind of anti-bot protection a site is using.
|
||||
|
||||
Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.
|
||||
Useful for debugging scraping issues.
|
||||
"""
|
||||
detector = BlockDetector()
|
||||
results = []
|
||||
|
||||
# Test direct
|
||||
try:
|
||||
client = await get_client()
|
||||
resp = await client.get(
|
||||
url,
|
||||
timeout=15,
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
)
|
||||
},
|
||||
)
|
||||
detection = detector.detect(resp.text, resp.status_code, dict(resp.headers))
|
||||
results.append({"method": "direct", "status": resp.status_code, **detection})
|
||||
except (httpx.HTTPError, httpx.RequestError) as e:
|
||||
results.append({"method": "direct", "error": str(e)})
|
||||
|
||||
# Test FlareSolverr
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as fs_client:
|
||||
fs_resp = await fs_client.post(
|
||||
settings.flaresolverr_url,
|
||||
json={"cmd": "request.get", "url": url, "maxTimeout": 15000},
|
||||
)
|
||||
if fs_resp.is_success:
|
||||
fs_data = fs_resp.json()
|
||||
fs_html = fs_data.get("solution", {}).get("response", "")
|
||||
fs_status = fs_data.get("solution", {}).get("status", 0)
|
||||
detection = detector.detect(fs_html, fs_status)
|
||||
results.append({"method": "flaresolverr", "status": fs_status, **detection})
|
||||
else:
|
||||
results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"})
|
||||
except (httpx.HTTPError, httpx.RequestError) as e:
|
||||
results.append({"method": "flaresolverr", "error": str(e)})
|
||||
|
||||
return {"success": True, "data": {"url": url, "results": results}}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system"
|
||||
)
|
||||
|
|
@ -788,74 +575,6 @@ async def capture_network(
|
|||
}
|
||||
|
||||
|
||||
@app.post("/v1/capture/lazy", tags=["Scraping"], summary="Detect and handle lazy-loaded content")
|
||||
async def detect_lazy_content(
|
||||
url: str = Body(...),
|
||||
auto_scroll: bool = Body(True),
|
||||
max_scrolls: int = Body(5),
|
||||
) -> dict[str, Any]:
|
||||
"""Detect lazy loading and infinite scroll patterns on a page.
|
||||
|
||||
Optionally generate JS to auto-scroll and load all content.
|
||||
"""
|
||||
from lazy_load import (
|
||||
detect_lazy_loading,
|
||||
generate_load_more_script,
|
||||
generate_scroll_script,
|
||||
)
|
||||
|
||||
result = await scraper.scrape(url, {"bypass_cloudflare": True})
|
||||
if result.get("status") != "ok":
|
||||
raise ScrapeError(result.get("error") or "Scrape failed")
|
||||
|
||||
html = result.get("raw_html", "")
|
||||
if not html:
|
||||
client = await get_client()
|
||||
try:
|
||||
resp = await client.get(
|
||||
url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"}
|
||||
)
|
||||
html = resp.text
|
||||
except (httpx.HTTPError, httpx.RequestError):
|
||||
raise ScrapeError("Could not fetch raw HTML") from None
|
||||
|
||||
detection = detect_lazy_loading(html)
|
||||
scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else ""
|
||||
load_more_script = generate_load_more_script() if auto_scroll else ""
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"url": url,
|
||||
"detection": detection,
|
||||
"has_lazy_content": any(detection.values()),
|
||||
"scroll_script": scroll_script,
|
||||
"load_more_script": load_more_script,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ── Crawl ──
|
||||
@app.post("/v1/crawl", tags=["Scraping"], summary="Crawl multiple pages from a URL")
|
||||
async def crawl(request: CrawlRequest) -> dict[str, Any]:
|
||||
"""Crawl multiple pages from a URL. Supports async webhooks."""
|
||||
if request.webhook:
|
||||
job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook)
|
||||
task = asyncio.create_task(_run_crawl_job(job_id, request))
|
||||
task.add_done_callback(_log_crawl_job_failure)
|
||||
return {"success": True, "data": {"id": job_id, "status": "pending"}}
|
||||
|
||||
pages = await scraper.crawl(
|
||||
request.url,
|
||||
{
|
||||
"max_pages": request.maxPages,
|
||||
"max_depth": request.maxDepth,
|
||||
"timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
|
||||
},
|
||||
)
|
||||
return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/crawl/adaptive",
|
||||
tags=["Scraping"],
|
||||
|
|
@ -929,28 +648,6 @@ async def adaptive_crawl(
|
|||
}
|
||||
|
||||
|
||||
async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None:
|
||||
try:
|
||||
pages = await scraper.crawl(
|
||||
request.url,
|
||||
{
|
||||
"max_pages": request.maxPages,
|
||||
"max_depth": request.maxDepth,
|
||||
},
|
||||
)
|
||||
await queue.complete_job(job_id, {"pages": pages})
|
||||
except Exception as e:
|
||||
logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url})
|
||||
await queue.fail_job(job_id, str(e))
|
||||
|
||||
|
||||
def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None:
|
||||
"""Log unhandled exceptions from crawl job tasks."""
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
logger.error("crawl_task_unhandled_error", extra={"error": str(exc)})
|
||||
|
||||
|
||||
async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any]) -> None:
|
||||
"""Fire a webhook notification for watch events."""
|
||||
try:
|
||||
|
|
@ -964,14 +661,6 @@ async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any
|
|||
logger.exception("watch_webhook_failed", extra={"url": url, "webhook": webhook})
|
||||
|
||||
|
||||
# ── Map ──
|
||||
@app.post("/v1/map", tags=["Scraping"], summary="Discover URLs on a site")
|
||||
async def map_pages(request: MapRequest) -> dict[str, Any]:
|
||||
"""Discover URLs on a site."""
|
||||
urls = await scraper.map_urls(request.url, {"limit": request.limit})
|
||||
return {"success": True, "data": {"links": urls}}
|
||||
|
||||
|
||||
# ── Parse (Documents) ──
|
||||
@app.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)")
|
||||
async def parse_document(request: ParseRequest) -> dict[str, Any]:
|
||||
|
|
@ -1392,31 +1081,6 @@ async def restore_session(session_id: str = Body(...)) -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
# ── Advanced Features (Firecrawl doesn't have these) ──
|
||||
|
||||
|
||||
@app.post("/v1/batch", tags=["Batch"], summary="Scrape multiple URLs in parallel")
|
||||
async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]:
|
||||
"""Scrape multiple URLs in parallel. Firecrawl charges extra for batch."""
|
||||
if len(urls) > 50:
|
||||
raise InvalidRequestError("Max 50 URLs per batch")
|
||||
tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
pages = []
|
||||
for i, r in enumerate(results):
|
||||
if isinstance(r, BaseException):
|
||||
pages.append({"url": urls[i], "error": str(r)})
|
||||
elif isinstance(r, dict):
|
||||
pages.append(
|
||||
{
|
||||
"url": urls[i],
|
||||
"markdown": r.get("content", ""),
|
||||
"method": r.get("method", "unknown"),
|
||||
}
|
||||
)
|
||||
return {"success": True, "data": {"pages": pages, "total": len(pages)}}
|
||||
|
||||
|
||||
@app.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs")
|
||||
async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]:
|
||||
"""Scrape two URLs and compare their content. Shows additions, deletions, changes."""
|
||||
|
|
@ -2244,7 +1908,6 @@ async def extract_stable(data: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|||
result = await scraper.scrape(url, {"bypass_cloudflare": True})
|
||||
if result.get("status") != "ok":
|
||||
raise ScrapeError(result.get("error") or "Extraction failed")
|
||||
from extractor import SchemaExtractor
|
||||
|
||||
ex = SchemaExtractor()
|
||||
extracted = await ex.extract(result.get("content", ""), fields, mode="llm")
|
||||
|
|
@ -3140,6 +2803,9 @@ async def list_schemas() -> dict[str, Any]:
|
|||
# (split into routers/auth.py on the api-router-split refactor)
|
||||
|
||||
app.include_router(auth_router)
|
||||
app.include_router(health_router)
|
||||
app.include_router(scraping_router)
|
||||
app.include_router(templates_router)
|
||||
|
||||
# ── Review ──
|
||||
|
||||
|
|
@ -3786,61 +3452,6 @@ async def detect_tech(url: str = Body(...)) -> dict[str, Any]:
|
|||
return {"success": True, "data": result.get("tech_stack", {})}
|
||||
|
||||
|
||||
# ── Scraper Templates ──
|
||||
|
||||
|
||||
@app.get("/v1/templates", tags=["Templates"], summary="List all pre-built scraper templates")
|
||||
async def list_templates_endpoint() -> dict[str, Any]:
|
||||
"""List all available pre-built scraper templates.
|
||||
|
||||
Templates are one-click extractors for popular websites:
|
||||
Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc.
|
||||
"""
|
||||
from template_engine import list_templates
|
||||
|
||||
templates = list_templates()
|
||||
# Group by category
|
||||
categories: dict[str, list[dict[str, Any]]] = {}
|
||||
for t in templates:
|
||||
cat = t.get("category", "general")
|
||||
categories.setdefault(cat, []).append(t)
|
||||
return {
|
||||
"success": True,
|
||||
"data": {"templates": templates, "categories": categories, "total": len(templates)},
|
||||
}
|
||||
|
||||
|
||||
@app.get("/v1/templates/{template_id}", tags=["Templates"], summary="Get a scraper template")
|
||||
async def get_template_endpoint(template_id: str) -> dict[str, Any]:
|
||||
"""Get a specific scraper template with full schema details."""
|
||||
from template_engine import get_template
|
||||
|
||||
template = get_template(template_id)
|
||||
if not template:
|
||||
raise NotFoundError(f"Template not found: {template_id}")
|
||||
return {"success": True, "data": template}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/v1/templates/execute", tags=["Templates"], summary="Execute a scraper template against a URL"
|
||||
)
|
||||
async def execute_template_endpoint(
|
||||
template_id: str = Body(...),
|
||||
url: str = Body(...),
|
||||
) -> dict[str, Any]:
|
||||
"""Execute a pre-built scraper template against any URL.
|
||||
|
||||
Example: use "amazon_product" template with an Amazon product URL
|
||||
to get structured title, price, rating, description, etc.
|
||||
|
||||
Templates auto-detect the page structure using pre-configured CSS selectors.
|
||||
"""
|
||||
from template_engine import execute_template
|
||||
|
||||
result = await execute_template(template_id, url)
|
||||
return result
|
||||
|
||||
|
||||
# ── AI Agent Integration ──
|
||||
|
||||
|
||||
|
|
|
|||
405
apify_schema.py
Normal file
405
apify_schema.py
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
"""Pry — Apify actor input/output schema builder.
|
||||
|
||||
Provides a fluent, typed API for constructing Apify actor input and output
|
||||
schemas (JSON Schema compatible with Apify's `INPUT_SCHEMA.json` and
|
||||
`OUTPUT_SCHEMA.json`).
|
||||
|
||||
Example:
|
||||
builder = ApifySchemaBuilder("Product Scraper")
|
||||
builder.add_string("url", "Start URL", required=True)
|
||||
builder.add_integer("maxResults", "Max Results", default=10, min=1, max=100)
|
||||
builder.add_enum("format", "Output Format", ["json", "csv", "markdown"], default="json")
|
||||
schema = builder.build_input_schema()
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class FieldSpec:
|
||||
"""Base class for Apify schema field specifications."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
field_type: str,
|
||||
description: str = "",
|
||||
default: Any | None = None,
|
||||
nullable: bool = False,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.title = title
|
||||
self.field_type = field_type
|
||||
self.description = description
|
||||
self.default = default
|
||||
self.nullable = nullable
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec: dict[str, Any] = {
|
||||
"title": self.title,
|
||||
"type": ([self.field_type, "null"] if self.nullable else self.field_type),
|
||||
}
|
||||
if self.description:
|
||||
spec["description"] = self.description
|
||||
if self.default is not None:
|
||||
spec["default"] = self.default
|
||||
return spec
|
||||
|
||||
|
||||
class StringField(FieldSpec):
|
||||
"""String field with optional pattern and editor hints."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
default: str | None = None,
|
||||
nullable: bool = False,
|
||||
min_length: int | None = None,
|
||||
max_length: int | None = None,
|
||||
pattern: str | None = None,
|
||||
editor: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(name, title, "string", description, default, nullable)
|
||||
self.min_length = min_length
|
||||
self.max_length = max_length
|
||||
self.pattern = pattern
|
||||
self.editor = editor
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec = super().to_dict()
|
||||
if self.min_length is not None:
|
||||
spec["minLength"] = self.min_length
|
||||
if self.max_length is not None:
|
||||
spec["maxLength"] = self.max_length
|
||||
if self.pattern is not None:
|
||||
spec["pattern"] = self.pattern
|
||||
if self.editor is not None:
|
||||
spec["editor"] = self.editor
|
||||
return spec
|
||||
|
||||
|
||||
class IntegerField(FieldSpec):
|
||||
"""Integer field with optional range."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
default: int | None = None,
|
||||
nullable: bool = False,
|
||||
minimum: int | None = None,
|
||||
maximum: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(name, title, "integer", description, default, nullable)
|
||||
self.minimum = minimum
|
||||
self.maximum = maximum
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec = super().to_dict()
|
||||
if self.minimum is not None:
|
||||
spec["minimum"] = self.minimum
|
||||
if self.maximum is not None:
|
||||
spec["maximum"] = self.maximum
|
||||
return spec
|
||||
|
||||
|
||||
class NumberField(FieldSpec):
|
||||
"""Number field with optional range."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
default: float | None = None,
|
||||
nullable: bool = False,
|
||||
minimum: float | None = None,
|
||||
maximum: float | None = None,
|
||||
) -> None:
|
||||
super().__init__(name, title, "number", description, default, nullable)
|
||||
self.minimum = minimum
|
||||
self.maximum = maximum
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec = super().to_dict()
|
||||
if self.minimum is not None:
|
||||
spec["minimum"] = self.minimum
|
||||
if self.maximum is not None:
|
||||
spec["maximum"] = self.maximum
|
||||
return spec
|
||||
|
||||
|
||||
class BooleanField(FieldSpec):
|
||||
"""Boolean field with optional default."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
default: bool | None = None,
|
||||
nullable: bool = False,
|
||||
) -> None:
|
||||
super().__init__(name, title, "boolean", description, default, nullable)
|
||||
|
||||
|
||||
class EnumField(FieldSpec):
|
||||
"""Enum field restricted to a set of allowed values."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
enum: list[str],
|
||||
description: str = "",
|
||||
default: str | None = None,
|
||||
nullable: bool = False,
|
||||
) -> None:
|
||||
super().__init__(name, title, "string", description, default, nullable)
|
||||
self.enum = enum
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec = super().to_dict()
|
||||
spec["enum"] = self.enum
|
||||
return spec
|
||||
|
||||
|
||||
class ArrayField(FieldSpec):
|
||||
"""Array field with item schema."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
items: dict[str, Any],
|
||||
description: str = "",
|
||||
default: list[Any] | None = None,
|
||||
nullable: bool = False,
|
||||
min_items: int | None = None,
|
||||
max_items: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(name, title, "array", description, default, nullable)
|
||||
self.items = items
|
||||
self.min_items = min_items
|
||||
self.max_items = max_items
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec = super().to_dict()
|
||||
spec["items"] = self.items
|
||||
if self.min_items is not None:
|
||||
spec["minItems"] = self.min_items
|
||||
if self.max_items is not None:
|
||||
spec["maxItems"] = self.max_items
|
||||
return spec
|
||||
|
||||
|
||||
class ObjectField(FieldSpec):
|
||||
"""Object field with nested properties."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
properties: dict[str, Any],
|
||||
description: str = "",
|
||||
default: dict[str, Any] | None = None,
|
||||
nullable: bool = False,
|
||||
required: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__(name, title, "object", description, default, nullable)
|
||||
self.properties = properties
|
||||
self.required = required or []
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
spec = super().to_dict()
|
||||
spec["properties"] = self.properties
|
||||
if self.required:
|
||||
spec["required"] = self.required
|
||||
return spec
|
||||
|
||||
|
||||
class ApifySchemaBuilder:
|
||||
"""Fluent builder for Apify actor input/output schemas."""
|
||||
|
||||
def __init__(self, title: str = "Actor Input", description: str = "", schema_version: int = 1) -> None:
|
||||
self.title = title
|
||||
self.description = description
|
||||
self.schema_version = schema_version
|
||||
self._fields: dict[str, FieldSpec] = {}
|
||||
self._required: list[str] = []
|
||||
self._prefab: str | None = None
|
||||
|
||||
def add_field(self, field: FieldSpec, required: bool = False) -> ApifySchemaBuilder:
|
||||
"""Add a field specification to the schema."""
|
||||
self._fields[field.name] = field
|
||||
if required:
|
||||
self._required.append(field.name)
|
||||
return self
|
||||
|
||||
def add_string(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: str | None = None,
|
||||
nullable: bool = False,
|
||||
min_length: int | None = None,
|
||||
max_length: int | None = None,
|
||||
pattern: str | None = None,
|
||||
editor: str | None = None,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = StringField(
|
||||
name,
|
||||
title,
|
||||
description=description,
|
||||
default=default,
|
||||
nullable=nullable,
|
||||
min_length=min_length,
|
||||
max_length=max_length,
|
||||
pattern=pattern,
|
||||
editor=editor,
|
||||
)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def add_integer(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: int | None = None,
|
||||
nullable: bool = False,
|
||||
minimum: int | None = None,
|
||||
maximum: int | None = None,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = IntegerField(
|
||||
name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum
|
||||
)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def add_number(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: float | None = None,
|
||||
nullable: bool = False,
|
||||
minimum: float | None = None,
|
||||
maximum: float | None = None,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = NumberField(
|
||||
name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum
|
||||
)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def add_boolean(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: bool | None = None,
|
||||
nullable: bool = False,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = BooleanField(name, title, description=description, default=default, nullable=nullable)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def add_enum(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
enum: list[str],
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: str | None = None,
|
||||
nullable: bool = False,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = EnumField(name, title, enum, description=description, default=default, nullable=nullable)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def add_array(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
items: dict[str, Any],
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: list[Any] | None = None,
|
||||
nullable: bool = False,
|
||||
min_items: int | None = None,
|
||||
max_items: int | None = None,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = ArrayField(
|
||||
name,
|
||||
title,
|
||||
items,
|
||||
description=description,
|
||||
default=default,
|
||||
nullable=nullable,
|
||||
min_items=min_items,
|
||||
max_items=max_items,
|
||||
)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def add_object(
|
||||
self,
|
||||
name: str,
|
||||
title: str,
|
||||
properties: dict[str, Any],
|
||||
description: str = "",
|
||||
required: bool = False,
|
||||
default: dict[str, Any] | None = None,
|
||||
nullable: bool = False,
|
||||
nested_required: list[str] | None = None,
|
||||
) -> ApifySchemaBuilder:
|
||||
field = ObjectField(
|
||||
name, title, properties, description=description, default=default, nullable=nullable, required=nested_required
|
||||
)
|
||||
return self.add_field(field, required=required)
|
||||
|
||||
def set_prefab(self, prefab: str) -> ApifySchemaBuilder:
|
||||
"""Set an Apify input schema prefab (e.g., 'START_URLS')."""
|
||||
self._prefab = prefab
|
||||
return self
|
||||
|
||||
def build_input_schema(self) -> dict[str, Any]:
|
||||
"""Build an Apify INPUT_SCHEMA.json-compatible dict."""
|
||||
schema: dict[str, Any] = {
|
||||
"title": self.title,
|
||||
"type": "object",
|
||||
"schemaVersion": self.schema_version,
|
||||
"properties": {name: field.to_dict() for name, field in self._fields.items()},
|
||||
}
|
||||
if self.description:
|
||||
schema["description"] = self.description
|
||||
if self._required:
|
||||
schema["required"] = self._required
|
||||
if self._prefab:
|
||||
schema["prefab"] = self._prefab
|
||||
return schema
|
||||
|
||||
def build_output_schema(self) -> dict[str, Any]:
|
||||
"""Build an Apify OUTPUT_SCHEMA.json-compatible dict."""
|
||||
schema: dict[str, Any] = {
|
||||
"title": self.title,
|
||||
"type": "object",
|
||||
"schemaVersion": self.schema_version,
|
||||
"properties": {name: field.to_dict() for name, field in self._fields.items()},
|
||||
}
|
||||
if self.description:
|
||||
schema["description"] = self.description
|
||||
if self._required:
|
||||
schema["required"] = self._required
|
||||
return schema
|
||||
36
deps.py
Normal file
36
deps.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""Shared global instances for Pry API.
|
||||
|
||||
This module initializes the singleton instances used by all API routers
|
||||
and should be imported by both api.py and any router module that needs
|
||||
direct access to shared state.
|
||||
|
||||
Usage:
|
||||
from deps import scraper, cache, automator, ...
|
||||
|
||||
Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
from __future__ import annotations
|
||||
|
||||
from advanced import PryAdvanced
|
||||
from automator import PryAutomator
|
||||
from cache import ResponseCache
|
||||
from extractor import SchemaExtractor
|
||||
from jobqueue import JobQueue
|
||||
from mconfig import PryConfig
|
||||
from parser import DocumentParser
|
||||
from ratelimit import RateLimiter
|
||||
from scraper import PryScraper
|
||||
|
||||
config = PryConfig()
|
||||
|
||||
scraper = PryScraper()
|
||||
automator = PryAutomator()
|
||||
parser = DocumentParser()
|
||||
extractor = SchemaExtractor()
|
||||
cache = ResponseCache(capacity=1000)
|
||||
ratelimiter = RateLimiter(default_rpm=120, burst=200)
|
||||
queue = JobQueue()
|
||||
advanced = PryAdvanced(cache=cache)
|
||||
|
|
@ -42,4 +42,8 @@ Licensed under MIT. See LICENSE.
|
|||
# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
|
||||
__all__: list[str] = [] # populated as routers are added
|
||||
__all__: list[str] = [
|
||||
"auth_router",
|
||||
"health_router",
|
||||
"templates_router",
|
||||
]
|
||||
|
|
|
|||
100
routers/health.py
Normal file
100
routers/health.py
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
"""Pry — Health router (liveness, readiness, dependency probes).
|
||||
|
||||
Split from api.py. Replaces the inline /health, /live, /ready endpoints.
|
||||
|
||||
Endpoints (3):
|
||||
GET /health — Full health check with dependency status
|
||||
GET /live — Kubernetes liveness probe
|
||||
GET /ready — Kubernetes readiness probe
|
||||
|
||||
Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import redis
|
||||
from fastapi import APIRouter
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from client import get_client
|
||||
from settings import settings
|
||||
|
||||
router = APIRouter(tags=["Health"])
|
||||
|
||||
|
||||
@router.get("/health", summary="Full health check with dependency status")
|
||||
async def health_check() -> JSONResponse:
|
||||
"""Comprehensive health check — probes Ollama, FlareSolverr, Redis."""
|
||||
deps = {"ollama": False, "flaresolverr": False, "redis": False}
|
||||
|
||||
async def check_ollama() -> bool:
|
||||
try:
|
||||
c = await get_client()
|
||||
r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3)
|
||||
return r.is_success
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
async def check_flare() -> bool:
|
||||
try:
|
||||
c = await get_client()
|
||||
r = await c.post(
|
||||
settings.flaresolverr_url,
|
||||
json={"cmd": "sessions.list"},
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=3,
|
||||
)
|
||||
return r.is_success
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
async def check_redis() -> bool:
|
||||
try:
|
||||
import redis.asyncio as aioredis
|
||||
|
||||
r = aioredis.from_url(settings.redis_url)
|
||||
await r.ping()
|
||||
await r.aclose()
|
||||
return True
|
||||
except redis.RedisError:
|
||||
return False
|
||||
|
||||
results = await asyncio.gather(
|
||||
check_ollama(), check_flare(), check_redis(), return_exceptions=False
|
||||
)
|
||||
deps["ollama"] = results[0]
|
||||
deps["flaresolverr"] = results[1]
|
||||
deps["redis"] = results[2]
|
||||
flaresolverr_ok = deps["flaresolverr"]
|
||||
status_code = 200 if flaresolverr_ok else 503
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"status": "ok" if flaresolverr_ok else "degraded",
|
||||
"version": "3.0.0",
|
||||
"dependencies": deps,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/live", summary="Kubernetes liveness probe")
|
||||
async def live() -> dict[str, str]:
|
||||
"""Simple liveness — always returns 200 if the process is running."""
|
||||
return {"status": "alive"}
|
||||
|
||||
|
||||
@router.get("/ready", summary="Kubernetes readiness probe", response_model=None)
|
||||
async def ready() -> JSONResponse | dict[str, str]:
|
||||
"""Readiness — checks critical dependencies."""
|
||||
try:
|
||||
c = await get_client()
|
||||
r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3)
|
||||
if r.is_success:
|
||||
return {"status": "ready"}
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return JSONResponse(status_code=503, content={"status": "not_ready"})
|
||||
301
routers/scraping.py
Normal file
301
routers/scraping.py
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
"""Pry — Scraping router (scrape, crawl, map, batch, detect).
|
||||
|
||||
Split from api.py on the api-router-split refactor. Replaces inline
|
||||
Scraping-tagged endpoints from api.py. Behavior is identical.
|
||||
|
||||
Endpoints (6):
|
||||
POST /v1/scrape — Scrape a single URL
|
||||
POST /v1/detect-block — Detect anti-bot protection
|
||||
POST /v1/capture/lazy — Detect lazy-loaded content
|
||||
POST /v1/crawl — Crawl multiple pages
|
||||
POST /v1/map — Discover URLs on a site
|
||||
POST /v1/batch — Scrape multiple URLs in parallel
|
||||
|
||||
Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Body
|
||||
from pydantic import BaseModel
|
||||
|
||||
from client import get_client
|
||||
from deps import cache, extractor, queue, scraper
|
||||
from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError
|
||||
from scraper import BlockDetector
|
||||
from settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(tags=["Scraping"])
|
||||
|
||||
|
||||
# ── Models ──
|
||||
|
||||
|
||||
class ScrapeRequest(BaseModel):
|
||||
url: str
|
||||
formats: list[str] | None = None
|
||||
onlyMainContent: bool | None = True
|
||||
timeout: int | None = 30
|
||||
bypassCloudflare: bool | None = True
|
||||
jsRender: bool | None = False
|
||||
jsonSchema: dict[str, str] | None = None
|
||||
|
||||
|
||||
class CrawlRequest(BaseModel):
|
||||
url: str
|
||||
maxPages: int | None = 10
|
||||
maxDepth: int | None = 2
|
||||
scrapeOptions: dict[str, Any] | None = None
|
||||
webhook: str | None = None
|
||||
|
||||
|
||||
class MapRequest(BaseModel):
|
||||
url: str
|
||||
search: str | None = None
|
||||
ignoreSitemap: bool | None = True
|
||||
limit: int | None = 50
|
||||
|
||||
|
||||
# ── Internal helpers ──
|
||||
|
||||
|
||||
async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None:
|
||||
try:
|
||||
pages = await scraper.crawl(
|
||||
request.url,
|
||||
{
|
||||
"max_pages": request.maxPages,
|
||||
"max_depth": request.maxDepth,
|
||||
},
|
||||
)
|
||||
await queue.complete_job(job_id, {"pages": pages})
|
||||
except Exception as e:
|
||||
logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url})
|
||||
await queue.fail_job(job_id, str(e))
|
||||
|
||||
|
||||
def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None:
|
||||
"""Log unhandled exceptions from crawl job tasks."""
|
||||
exc = task.exception()
|
||||
if exc:
|
||||
logger.error("crawl_task_unhandled_error", extra={"error": str(exc)})
|
||||
|
||||
|
||||
# ── Scrape ──
|
||||
|
||||
|
||||
@router.post("/v1/scrape", summary="Scrape a single URL")
|
||||
async def scrape(request: ScrapeRequest) -> dict[str, Any]:
|
||||
"""Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON."""
|
||||
# Check cache
|
||||
cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender}
|
||||
cached = cache.get(request.url, cache_opts)
|
||||
if cached:
|
||||
cached["_cached"] = True
|
||||
return cached
|
||||
|
||||
try:
|
||||
result = await scraper.scrape(
|
||||
request.url,
|
||||
{
|
||||
"timeout": request.timeout,
|
||||
"bypass_cloudflare": request.bypassCloudflare,
|
||||
"js_render": request.jsRender,
|
||||
"formats": request.formats,
|
||||
},
|
||||
)
|
||||
if result.get("status") != "ok":
|
||||
raise ScrapeError(result.get("error", "Scrape failed"))
|
||||
|
||||
response: dict[str, Any] = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": result.get("content", ""),
|
||||
"metadata": {
|
||||
"url": request.url,
|
||||
"method": result.get("method", "unknown"),
|
||||
"title": result.get("title", ""),
|
||||
"description": result.get("description", ""),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
# JSON schema extraction if requested
|
||||
if request.jsonSchema:
|
||||
extracted = await extractor.extract(result.get("content", ""), request.jsonSchema)
|
||||
response["data"]["json"] = extracted
|
||||
|
||||
cache.set(request.url, response, cache_opts)
|
||||
return response
|
||||
except PryError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise ExternalServiceError(str(e)) from e
|
||||
|
||||
|
||||
@router.post("/v1/detect-block", summary="Detect if a site is blocking the scraper")
|
||||
async def detect_block(url: str = Body(...)) -> dict[str, Any]:
|
||||
"""Detect what kind of anti-bot protection a site is using.
|
||||
|
||||
Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.
|
||||
Useful for debugging scraping issues.
|
||||
"""
|
||||
detector = BlockDetector()
|
||||
results = []
|
||||
|
||||
# Test direct
|
||||
try:
|
||||
client = await get_client()
|
||||
resp = await client.get(
|
||||
url,
|
||||
timeout=15,
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/125.0.0.0 Safari/537.36"
|
||||
)
|
||||
},
|
||||
)
|
||||
detection = detector.detect(resp.text, resp.status_code, dict(resp.headers))
|
||||
results.append({"method": "direct", "status": resp.status_code, **detection})
|
||||
except (httpx.HTTPError, httpx.RequestError) as e:
|
||||
results.append({"method": "direct", "error": str(e)})
|
||||
|
||||
# Test FlareSolverr
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as fs_client:
|
||||
fs_resp = await fs_client.post(
|
||||
settings.flaresolverr_url,
|
||||
json={"cmd": "request.get", "url": url, "maxTimeout": 15000},
|
||||
)
|
||||
if fs_resp.is_success:
|
||||
fs_data = fs_resp.json()
|
||||
fs_html = fs_data.get("solution", {}).get("response", "")
|
||||
fs_status = fs_data.get("solution", {}).get("status", 0)
|
||||
detection = detector.detect(fs_html, fs_status)
|
||||
results.append({"method": "flaresolverr", "status": fs_status, **detection})
|
||||
else:
|
||||
results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"})
|
||||
except (httpx.HTTPError, httpx.RequestError) as e:
|
||||
results.append({"method": "flaresolverr", "error": str(e)})
|
||||
|
||||
return {"success": True, "data": {"url": url, "results": results}}
|
||||
|
||||
|
||||
@router.post("/v1/capture/lazy", summary="Detect and handle lazy-loaded content")
|
||||
async def detect_lazy_content(
|
||||
url: str = Body(...),
|
||||
auto_scroll: bool = Body(True),
|
||||
max_scrolls: int = Body(5),
|
||||
) -> dict[str, Any]:
|
||||
"""Detect lazy loading and infinite scroll patterns on a page.
|
||||
|
||||
Optionally generate JS to auto-scroll and load all content.
|
||||
"""
|
||||
from lazy_load import (
|
||||
detect_lazy_loading,
|
||||
generate_load_more_script,
|
||||
generate_scroll_script,
|
||||
)
|
||||
|
||||
result = await scraper.scrape(url, {"bypass_cloudflare": True})
|
||||
if result.get("status") != "ok":
|
||||
raise ScrapeError(result.get("error") or "Scrape failed")
|
||||
|
||||
html = result.get("raw_html", "")
|
||||
if not html:
|
||||
client = await get_client()
|
||||
try:
|
||||
resp = await client.get(
|
||||
url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"}
|
||||
)
|
||||
html = resp.text
|
||||
except (httpx.HTTPError, httpx.RequestError):
|
||||
raise ScrapeError("Could not fetch raw HTML") from None
|
||||
|
||||
detection = detect_lazy_loading(html)
|
||||
scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else ""
|
||||
load_more_script = generate_load_more_script() if auto_scroll else ""
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"url": url,
|
||||
"detection": detection,
|
||||
"has_lazy_content": any(detection.values()),
|
||||
"scroll_script": scroll_script,
|
||||
"load_more_script": load_more_script,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ── Crawl ──
|
||||
|
||||
|
||||
@router.post("/v1/crawl", summary="Crawl multiple pages from a URL")
|
||||
async def crawl(request: CrawlRequest) -> dict[str, Any]:
|
||||
"""Crawl multiple pages from a URL. Supports async webhooks."""
|
||||
if request.webhook:
|
||||
job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook)
|
||||
task = asyncio.create_task(_run_crawl_job(job_id, request))
|
||||
task.add_done_callback(_log_crawl_job_failure)
|
||||
return {"success": True, "data": {"id": job_id, "status": "pending"}}
|
||||
|
||||
pages = await scraper.crawl(
|
||||
request.url,
|
||||
{
|
||||
"max_pages": request.maxPages,
|
||||
"max_depth": request.maxDepth,
|
||||
"timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
|
||||
},
|
||||
)
|
||||
return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}}
|
||||
|
||||
|
||||
# ── Map ──
|
||||
|
||||
|
||||
@router.post("/v1/map", summary="Discover URLs on a site")
|
||||
async def map_pages(request: MapRequest) -> dict[str, Any]:
|
||||
"""Discover URLs on a site."""
|
||||
urls = await scraper.map_urls(request.url, {"limit": request.limit})
|
||||
return {"success": True, "data": {"links": urls}}
|
||||
|
||||
|
||||
# ── Batch ──
|
||||
|
||||
|
||||
@router.post("/v1/batch", summary="Scrape multiple URLs in parallel")
|
||||
async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]:
|
||||
"""Scrape multiple URLs in parallel. Firecrawl charges extra for batch."""
|
||||
if len(urls) > 50:
|
||||
raise InvalidRequestError("Max 50 URLs per batch")
|
||||
tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
pages = []
|
||||
for i, r in enumerate(results):
|
||||
if isinstance(r, BaseException):
|
||||
pages.append({"url": urls[i], "error": str(r)})
|
||||
elif isinstance(r, dict):
|
||||
pages.append(
|
||||
{
|
||||
"url": urls[i],
|
||||
"markdown": r.get("content", ""),
|
||||
"method": r.get("method", "unknown"),
|
||||
}
|
||||
)
|
||||
return {"success": True, "data": {"pages": pages, "total": len(pages)}}
|
||||
131
routers/templates.py
Normal file
131
routers/templates.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
"""Pry — Templates router."""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Body
|
||||
from pydantic import BaseModel
|
||||
|
||||
from errors import InvalidRequestError, NotFoundError
|
||||
from template_engine import execute_template, get_template, list_templates
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(tags=["Templates"])
|
||||
|
||||
|
||||
class BatchItem(BaseModel):
|
||||
"""A single template execution request within a batch."""
|
||||
|
||||
template_id: str
|
||||
url: str
|
||||
options: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class BatchResponse(BaseModel):
|
||||
"""Batch execution result."""
|
||||
|
||||
success: bool = True
|
||||
data: dict[str, Any]
|
||||
|
||||
|
||||
MAX_BATCH_SIZE = 20
|
||||
|
||||
|
||||
@router.get("/v1/templates", summary="List all pre-built scraper templates")
|
||||
async def list_templates_endpoint() -> dict[str, Any]:
|
||||
"""List all available pre-built scraper templates.
|
||||
|
||||
Templates are one-click extractors for popular websites:
|
||||
Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc.
|
||||
"""
|
||||
templates = list_templates()
|
||||
categories: dict[str, list[dict[str, Any]]] = {}
|
||||
for t in templates:
|
||||
cat = t.get("category", "general")
|
||||
categories.setdefault(cat, []).append(t)
|
||||
return {
|
||||
"success": True,
|
||||
"data": {"templates": templates, "categories": categories, "total": len(templates)},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/v1/templates/{template_id}", summary="Get a scraper template")
|
||||
async def get_template_endpoint(template_id: str) -> dict[str, Any]:
|
||||
"""Get a specific scraper template with full schema details."""
|
||||
template = get_template(template_id)
|
||||
if not template:
|
||||
raise NotFoundError(f"Template not found: {template_id}")
|
||||
return {"success": True, "data": template}
|
||||
|
||||
|
||||
@router.post("/v1/templates/execute", summary="Execute a scraper template against a URL")
|
||||
async def execute_template_endpoint(
|
||||
template_id: str = Body(...),
|
||||
url: str = Body(...),
|
||||
) -> dict[str, Any]:
|
||||
"""Execute a pre-built scraper template against any URL.
|
||||
|
||||
Example: use "amazon_product" template with an Amazon product URL
|
||||
to get structured title, price, rating, description, etc.
|
||||
|
||||
Templates auto-detect the page structure using pre-configured CSS selectors.
|
||||
"""
|
||||
result = await execute_template(template_id, url)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/v1/templates/batch", summary="Execute multiple templates/URLs in parallel")
|
||||
async def batch_execute(body: list[BatchItem]) -> dict[str, Any]:
|
||||
"""Execute multiple scraper templates against multiple URLs in parallel.
|
||||
|
||||
Runs up to 20 template executions concurrently. Each result preserves the
|
||||
original template_id and url for correlation.
|
||||
"""
|
||||
if len(body) > MAX_BATCH_SIZE:
|
||||
raise InvalidRequestError(f"Max {MAX_BATCH_SIZE} items per batch, got {len(body)}")
|
||||
|
||||
async def run_item(item: BatchItem) -> dict[str, Any]:
|
||||
try:
|
||||
result = await execute_template(item.template_id, item.url, **(item.options or {}))
|
||||
return {
|
||||
"template_id": item.template_id,
|
||||
"url": item.url,
|
||||
"result": result,
|
||||
}
|
||||
except Exception as e: # noqa: BLE001
|
||||
return {
|
||||
"template_id": item.template_id,
|
||||
"url": item.url,
|
||||
"error": {"code": "execution_failed", "message": str(e)},
|
||||
}
|
||||
|
||||
results = await asyncio.gather(*[run_item(item) for item in body], return_exceptions=True)
|
||||
|
||||
flat: list[dict[str, Any]] = []
|
||||
failed = 0
|
||||
for r in results:
|
||||
if isinstance(r, Exception):
|
||||
flat.append({"error": {"code": "execution_failed", "message": str(r)}})
|
||||
failed += 1
|
||||
else:
|
||||
if "error" in r:
|
||||
failed += 1
|
||||
flat.append(r)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"results": flat,
|
||||
"total": len(flat),
|
||||
"failed": failed,
|
||||
},
|
||||
}
|
||||
|
|
@ -2,14 +2,26 @@
|
|||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
"""Tests for API helpers."""
|
||||
# Licensed under MIT.
|
||||
"""Tests for API helpers and auth."""
|
||||
|
||||
from api import _get_or_key
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_get_or_key_from_env(monkeypatch) -> None:
|
||||
monkeypatch.setattr("api.settings.openrouter_api_key", "sk-test-key-123")
|
||||
from api import _get_or_key
|
||||
|
||||
key = _get_or_key()
|
||||
assert key == "sk-test-key-123"
|
||||
|
||||
|
|
@ -17,6 +29,8 @@ def test_get_or_key_from_env(monkeypatch) -> None:
|
|||
def test_get_or_key_empty_when_not_set(monkeypatch) -> None:
|
||||
monkeypatch.setattr("api.settings.openrouter_api_key", "")
|
||||
monkeypatch.setattr("os.path.isfile", lambda p: False)
|
||||
from api import _get_or_key
|
||||
|
||||
key = _get_or_key()
|
||||
assert key is None
|
||||
|
||||
|
|
@ -26,3 +40,31 @@ def test_vision_models_fallback_list() -> None:
|
|||
|
||||
assert len(VISION_MODELS_FALLBACK) >= 3
|
||||
assert all("free" in m for m in VISION_MODELS_FALLBACK)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> None:
|
||||
"""Protected endpoints require a valid Bearer PRY_API_KEY."""
|
||||
monkeypatch.setattr("api.settings.api_key", "secret-pry-key")
|
||||
# Disable x402 payment gating so we can test auth in isolation.
|
||||
with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
|
||||
resp = client.post("/v1/scrape", json={"url": "https://example.com"})
|
||||
assert resp.status_code == 401
|
||||
|
||||
with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
|
||||
resp = client.post(
|
||||
"/v1/scrape",
|
||||
json={"url": "https://example.com"},
|
||||
headers={"authorization": "Bearer wrong-key"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
|
||||
resp = client.post(
|
||||
"/v1/scrape",
|
||||
json={"url": "https://example.com"},
|
||||
headers={"authorization": "Bearer secret-pry-key"},
|
||||
)
|
||||
# auth passes; endpoint returns 402 because we did not mock scraper, which is fine
|
||||
assert resp.status_code in (200, 402)
|
||||
|
||||
|
|
|
|||
162
tests/test_api_scraping.py
Normal file
162
tests/test_api_scraping.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT.
|
||||
"""End-to-end tests for the scraping router endpoints.
|
||||
|
||||
These tests exercise the full FastAPI request/response lifecycle via
|
||||
TestClient. External I/O (HTTP requests, browser automation) is mocked at the
|
||||
`deps.scraper` and `deps.extractor` boundaries so tests stay fast and
|
||||
deterministic.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api import app
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def bypass_x402_middleware() -> None:
|
||||
"""Disable x402 payment gating so scraping endpoints can be tested."""
|
||||
with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_response_cache() -> None:
|
||||
"""Clear the shared response cache so tests don't see each other's data."""
|
||||
from deps import cache as response_cache
|
||||
|
||||
response_cache.clear()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_scrape_result() -> dict:
|
||||
return {
|
||||
"status": "ok",
|
||||
"url": "https://example.com",
|
||||
"title": "Example",
|
||||
"description": "An example page",
|
||||
"content": "Hello world",
|
||||
"markdown": "# Hello world",
|
||||
"raw_html": "<html><body>Hello world</body></html>",
|
||||
"links": ["https://example.com/a", "https://example.com/b"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_scrape_success(client: TestClient, mock_scrape_result: dict) -> None:
|
||||
with patch("routers.scraping.scraper") as mock_scraper:
|
||||
mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result)
|
||||
resp = client.post("/v1/scrape", json={"url": "https://example.com"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["metadata"]["url"] == "https://example.com"
|
||||
assert data["data"]["markdown"] == "Hello world"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_scrape_with_json_schema(client: TestClient, mock_scrape_result: dict) -> None:
|
||||
schema = {"title": "page title"}
|
||||
extracted = {"title": "Example"}
|
||||
with patch("routers.scraping.scraper") as mock_scraper, patch("routers.scraping.extractor") as mock_extractor:
|
||||
mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result)
|
||||
mock_extractor.extract = AsyncMock(return_value=extracted)
|
||||
resp = client.post("/v1/scrape", json={"url": "https://example.com", "jsonSchema": schema})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["data"]["json"] == extracted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_detect_block_success(client: TestClient) -> None:
|
||||
html = "<html><body>ok</body></html>"
|
||||
with patch("routers.scraping.get_client") as mock_get_client:
|
||||
mock_client = AsyncMock()
|
||||
mock_client.get = AsyncMock(
|
||||
return_value=AsyncMock(
|
||||
text=html,
|
||||
status_code=200,
|
||||
headers={"content-type": "text/html"},
|
||||
)
|
||||
)
|
||||
mock_get_client.return_value = mock_client
|
||||
resp = client.post("/v1/detect-block", content="\"https://example.com\"", headers={"content-type": "application/json"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["url"] == "https://example.com"
|
||||
assert any(r["method"] == "direct" for r in data["data"]["results"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_capture_lazy_success(client: TestClient, mock_scrape_result: dict) -> None:
|
||||
with patch("routers.scraping.scraper") as mock_scraper:
|
||||
mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result)
|
||||
resp = client.post("/v1/capture/lazy", json={"url": "https://example.com"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert "detection" in data["data"]
|
||||
assert "has_lazy_content" in data["data"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_crawl_success(client: TestClient, mock_scrape_result: dict) -> None:
|
||||
pages = [mock_scrape_result]
|
||||
with patch("routers.scraping.scraper") as mock_scraper:
|
||||
mock_scraper.crawl = AsyncMock(return_value=pages)
|
||||
resp = client.post("/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert "pages" in data["data"]
|
||||
assert data["data"]["url"] == "https://example.com"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_map_success(client: TestClient) -> None:
|
||||
urls = ["https://example.com/page1", "https://example.com/page2"]
|
||||
with patch("routers.scraping.scraper") as mock_scraper:
|
||||
mock_scraper.map_urls = AsyncMock(return_value=urls)
|
||||
resp = client.post("/v1/map", json={"url": "https://example.com", "limit": 10})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["links"] == urls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_batch_success(client: TestClient, mock_scrape_result: dict) -> None:
|
||||
urls = ["https://example.com/1", "https://example.com/2"]
|
||||
with patch("routers.scraping.scraper") as mock_scraper:
|
||||
mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result)
|
||||
resp = client.post("/v1/batch", json=urls)
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert len(data["data"]["pages"]) == len(urls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_batch_requires_urls(client: TestClient) -> None:
|
||||
resp = client.post("/v1/batch", json={"not_a_list": True})
|
||||
assert resp.status_code == 422
|
||||
99
tests/test_api_templates.py
Normal file
99
tests/test_api_templates.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT.
|
||||
"""End-to-end tests for the templates router endpoints."""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from api import app
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def bypass_x402_middleware() -> None:
|
||||
"""Disable x402 payment gating for template execution tests."""
|
||||
with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client() -> TestClient:
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_templates(client: TestClient) -> None:
|
||||
templates = [
|
||||
{"id": "amazon_product", "name": "Amazon Product", "category": "ecommerce"},
|
||||
{"id": "github_repo", "name": "GitHub Repo", "category": "dev"},
|
||||
]
|
||||
with patch("routers.templates.list_templates", return_value=templates):
|
||||
resp = client.get("/v1/templates")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["total"] == 2
|
||||
assert "ecommerce" in data["data"]["categories"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_template(client: TestClient) -> None:
|
||||
template = {"id": "amazon_product", "name": "Amazon Product"}
|
||||
with patch("routers.templates.get_template", return_value=template):
|
||||
resp = client.get("/v1/templates/amazon_product")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["id"] == "amazon_product"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_template_not_found(client: TestClient) -> None:
|
||||
with patch("routers.templates.get_template", return_value=None):
|
||||
resp = client.get("/v1/templates/missing")
|
||||
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_template(client: TestClient) -> None:
|
||||
result = {"success": True, "data": {"title": "Widget"}}
|
||||
with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)):
|
||||
resp = client.post("/v1/templates/execute", json={"template_id": "amazon_product", "url": "https://example.com"})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["title"] == "Widget"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_execute_templates(client: TestClient) -> None:
|
||||
result = {"success": True, "data": {"title": "Widget"}}
|
||||
items = [
|
||||
{"template_id": "amazon_product", "url": "https://example.com/1"},
|
||||
{"template_id": "github_repo", "url": "https://example.com/2"},
|
||||
]
|
||||
with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)):
|
||||
resp = client.post("/v1/templates/batch", json=items)
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["success"] is True
|
||||
assert data["data"]["total"] == 2
|
||||
assert data["data"]["failed"] == 0
|
||||
assert data["data"]["results"][0]["template_id"] == "amazon_product"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_execute_templates_limits_size(client: TestClient) -> None:
|
||||
items = [{"template_id": "t", "url": "https://example.com"} for _ in range(21)]
|
||||
resp = client.post("/v1/templates/batch", json=items)
|
||||
|
||||
assert resp.status_code == 400
|
||||
72
tests/test_apify_schema.py
Normal file
72
tests/test_apify_schema.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT.
|
||||
"""Tests for the Apify schema builder."""
|
||||
|
||||
from apify_schema import ApifySchemaBuilder
|
||||
|
||||
|
||||
def test_input_schema_basic() -> None:
|
||||
schema = (
|
||||
ApifySchemaBuilder("Product Scraper", "Scrape product details")
|
||||
.add_string("url", "Start URL", description="The URL to scrape", required=True)
|
||||
.add_integer("maxResults", "Max Results", default=10, minimum=1, maximum=100)
|
||||
.add_enum("format", "Output Format", ["json", "csv", "markdown"], default="json")
|
||||
.build_input_schema()
|
||||
)
|
||||
|
||||
assert schema["title"] == "Product Scraper"
|
||||
assert schema["description"] == "Scrape product details"
|
||||
assert schema["type"] == "object"
|
||||
assert schema["schemaVersion"] == 1
|
||||
assert schema["required"] == ["url"]
|
||||
assert schema["properties"]["url"]["type"] == "string"
|
||||
assert schema["properties"]["maxResults"]["default"] == 10
|
||||
assert schema["properties"]["format"]["enum"] == ["json", "csv", "markdown"]
|
||||
|
||||
|
||||
def test_output_schema_includes_required() -> None:
|
||||
schema = (
|
||||
ApifySchemaBuilder("Product Output")
|
||||
.add_string("title", "Title", required=True)
|
||||
.add_number("price", "Price", minimum=0)
|
||||
.add_boolean("inStock", "In Stock", default=True)
|
||||
.build_output_schema()
|
||||
)
|
||||
|
||||
assert schema["properties"]["title"]["type"] == "string"
|
||||
assert schema["properties"]["price"]["minimum"] == 0
|
||||
assert schema["properties"]["inStock"]["default"] is True
|
||||
assert schema["required"] == ["title"]
|
||||
|
||||
|
||||
def test_array_and_object_fields() -> None:
|
||||
schema = (
|
||||
ApifySchemaBuilder("Complex Actor")
|
||||
.add_array("urls", "URLs", {"type": "string"}, required=True, min_items=1, max_items=50)
|
||||
.add_object(
|
||||
"proxy",
|
||||
"Proxy Config",
|
||||
{"host": {"type": "string"}, "port": {"type": "integer"}},
|
||||
nested_required=["host"],
|
||||
)
|
||||
.build_input_schema()
|
||||
)
|
||||
|
||||
assert schema["properties"]["urls"]["type"] == "array"
|
||||
assert schema["properties"]["urls"]["minItems"] == 1
|
||||
assert schema["properties"]["urls"]["items"]["type"] == "string"
|
||||
assert schema["properties"]["proxy"]["type"] == "object"
|
||||
assert schema["properties"]["proxy"]["required"] == ["host"]
|
||||
|
||||
|
||||
def test_nullable_field() -> None:
|
||||
schema = ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema()
|
||||
assert schema["properties"]["optional"]["type"] == ["string", "null"]
|
||||
|
||||
|
||||
def test_prefab() -> None:
|
||||
schema = ApifySchemaBuilder().set_prefab("START_URLS").add_string("url", "URL", required=True).build_input_schema()
|
||||
assert schema["prefab"] == "START_URLS"
|
||||
1
x402.py
1
x402.py
|
|
@ -48,6 +48,7 @@ X402_PRICING: dict[str, dict[str, Any]] = {
|
|||
"monitor": {"price_usd": 0.02, "description": "Create scheduled monitor"},
|
||||
"llm_call": {"price_usd": 0.01, "description": "LLM extraction call"},
|
||||
"template_execute": {"price_usd": 0.002, "description": "Execute scraper template"},
|
||||
"template_batch_execute": {"price_usd": 0.01, "description": "Execute up to 20 template items"},
|
||||
"browser_automation": {"price_usd": 0.05, "description": "Browser automation"},
|
||||
"bulk_crawl": {"price_usd": 0.10, "description": "Crawl up to 1000 pages"},
|
||||
"pdf_extract": {"price_usd": 0.01, "description": "PDF table extraction"},
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ PAID_ENDPOINTS = {
|
|||
"/v1/extract/css": "extract",
|
||||
"/v1/extract/llm": "llm_call",
|
||||
"/v1/templates/execute": "template_execute",
|
||||
"/v1/templates/batch": "template_batch_execute",
|
||||
"/v1/automate": "browser_automation",
|
||||
"/v1/monitor": "monitor",
|
||||
"/v1/pdf/extract": "pdf_extract",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue