feat(db): add Alembic migrations (#6)

routers/__init__.py

@@ -42,4 +42,8 @@ Licensed under MIT. See LICENSE.
 # Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
 # Licensed under MIT. See LICENSE.
 
-__all__: list[str] = []  # populated as routers are added
+__all__: list[str] = [
+    "auth_router",
+    "health_router",
+    "templates_router",
+]

routers/health.py

@@ -0,0 +1,100 @@
+"""Pry — Health router (liveness, readiness, dependency probes).
+
+Split from api.py. Replaces the inline /health, /live, /ready endpoints.
+
+Endpoints (3):
+  GET /health    — Full health check with dependency status
+  GET /live      — Kubernetes liveness probe
+  GET /ready     — Kubernetes readiness probe
+
+Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+from __future__ import annotations
+
+import asyncio
+
+import redis
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+
+from client import get_client
+from settings import settings
+
+router = APIRouter(tags=["Health"])
+
+
+@router.get("/health", summary="Full health check with dependency status")
+async def health_check() -> JSONResponse:
+    """Comprehensive health check — probes Ollama, FlareSolverr, Redis."""
+    deps = {"ollama": False, "flaresolverr": False, "redis": False}
+
+    async def check_ollama() -> bool:
+        try:
+            c = await get_client()
+            r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3)
+            return r.is_success
+        except Exception:  # noqa: BLE001
+            return False
+
+    async def check_flare() -> bool:
+        try:
+            c = await get_client()
+            r = await c.post(
+                settings.flaresolverr_url,
+                json={"cmd": "sessions.list"},
+                headers={"Content-Type": "application/json"},
+                timeout=3,
+            )
+            return r.is_success
+        except Exception:  # noqa: BLE001
+            return False
+
+    async def check_redis() -> bool:
+        try:
+            import redis.asyncio as aioredis
+
+            r = aioredis.from_url(settings.redis_url)
+            await r.ping()
+            await r.aclose()
+            return True
+        except redis.RedisError:
+            return False
+
+    results = await asyncio.gather(
+        check_ollama(), check_flare(), check_redis(), return_exceptions=False
+    )
+    deps["ollama"] = results[0]
+    deps["flaresolverr"] = results[1]
+    deps["redis"] = results[2]
+    flaresolverr_ok = deps["flaresolverr"]
+    status_code = 200 if flaresolverr_ok else 503
+    return JSONResponse(
+        status_code=status_code,
+        content={
+            "status": "ok" if flaresolverr_ok else "degraded",
+            "version": "3.0.0",
+            "dependencies": deps,
+        },
+    )
+
+
+@router.get("/live", summary="Kubernetes liveness probe")
+async def live() -> dict[str, str]:
+    """Simple liveness — always returns 200 if the process is running."""
+    return {"status": "alive"}
+
+
+@router.get("/ready", summary="Kubernetes readiness probe", response_model=None)
+async def ready() -> JSONResponse | dict[str, str]:
+    """Readiness — checks critical dependencies."""
+    try:
+        c = await get_client()
+        r = await c.get(f"{settings.ollama_url}/api/tags", timeout=3)
+        if r.is_success:
+            return {"status": "ready"}
+    except Exception:  # noqa: BLE001
+        pass
+    return JSONResponse(status_code=503, content={"status": "not_ready"})

routers/scraping.py

@@ -0,0 +1,301 @@
+"""Pry — Scraping router (scrape, crawl, map, batch, detect).
+
+Split from api.py on the api-router-split refactor. Replaces inline
+Scraping-tagged endpoints from api.py. Behavior is identical.
+
+Endpoints (6):
+  POST /v1/scrape          — Scrape a single URL
+  POST /v1/detect-block    — Detect anti-bot protection
+  POST /v1/capture/lazy    — Detect lazy-loaded content
+  POST /v1/crawl           — Crawl multiple pages
+  POST /v1/map             — Discover URLs on a site
+  POST /v1/batch           — Scrape multiple URLs in parallel
+
+Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Body
+from pydantic import BaseModel
+
+from client import get_client
+from deps import cache, extractor, queue, scraper
+from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError
+from scraper import BlockDetector
+from settings import settings
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Scraping"])
+
+
+# ── Models ──
+
+
+class ScrapeRequest(BaseModel):
+    url: str
+    formats: list[str] | None = None
+    onlyMainContent: bool | None = True
+    timeout: int | None = 30
+    bypassCloudflare: bool | None = True
+    jsRender: bool | None = False
+    jsonSchema: dict[str, str] | None = None
+
+
+class CrawlRequest(BaseModel):
+    url: str
+    maxPages: int | None = 10
+    maxDepth: int | None = 2
+    scrapeOptions: dict[str, Any] | None = None
+    webhook: str | None = None
+
+
+class MapRequest(BaseModel):
+    url: str
+    search: str | None = None
+    ignoreSitemap: bool | None = True
+    limit: int | None = 50
+
+
+# ── Internal helpers ──
+
+
+async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None:
+    try:
+        pages = await scraper.crawl(
+            request.url,
+            {
+                "max_pages": request.maxPages,
+                "max_depth": request.maxDepth,
+            },
+        )
+        await queue.complete_job(job_id, {"pages": pages})
+    except Exception as e:
+        logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url})
+        await queue.fail_job(job_id, str(e))
+
+
+def _log_crawl_job_failure(task: asyncio.Task[Any]) -> None:
+    """Log unhandled exceptions from crawl job tasks."""
+    exc = task.exception()
+    if exc:
+        logger.error("crawl_task_unhandled_error", extra={"error": str(exc)})
+
+
+# ── Scrape ──
+
+
+@router.post("/v1/scrape", summary="Scrape a single URL")
+async def scrape(request: ScrapeRequest) -> dict[str, Any]:
+    """Scrape a URL. Auto-bypasses Cloudflare. Returns markdown or JSON."""
+    # Check cache
+    cache_opts = {"bypass_cloudflare": request.bypassCloudflare, "js_render": request.jsRender}
+    cached = cache.get(request.url, cache_opts)
+    if cached:
+        cached["_cached"] = True
+        return cached
+
+    try:
+        result = await scraper.scrape(
+            request.url,
+            {
+                "timeout": request.timeout,
+                "bypass_cloudflare": request.bypassCloudflare,
+                "js_render": request.jsRender,
+                "formats": request.formats,
+            },
+        )
+        if result.get("status") != "ok":
+            raise ScrapeError(result.get("error", "Scrape failed"))
+
+        response: dict[str, Any] = {
+            "success": True,
+            "data": {
+                "markdown": result.get("content", ""),
+                "metadata": {
+                    "url": request.url,
+                    "method": result.get("method", "unknown"),
+                    "title": result.get("title", ""),
+                    "description": result.get("description", ""),
+                },
+            },
+        }
+
+        # JSON schema extraction if requested
+        if request.jsonSchema:
+            extracted = await extractor.extract(result.get("content", ""), request.jsonSchema)
+            response["data"]["json"] = extracted
+
+        cache.set(request.url, response, cache_opts)
+        return response
+    except PryError:
+        raise
+    except Exception as e:
+        raise ExternalServiceError(str(e)) from e
+
+
+@router.post("/v1/detect-block", summary="Detect if a site is blocking the scraper")
+async def detect_block(url: str = Body(...)) -> dict[str, Any]:
+    """Detect what kind of anti-bot protection a site is using.
+
+    Returns detection tier, vendor (Cloudflare/DataDome/etc.), and confidence.
+    Useful for debugging scraping issues.
+    """
+    detector = BlockDetector()
+    results = []
+
+    # Test direct
+    try:
+        client = await get_client()
+        resp = await client.get(
+            url,
+            timeout=15,
+            follow_redirects=True,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/125.0.0.0 Safari/537.36"
+                )
+            },
+        )
+        detection = detector.detect(resp.text, resp.status_code, dict(resp.headers))
+        results.append({"method": "direct", "status": resp.status_code, **detection})
+    except (httpx.HTTPError, httpx.RequestError) as e:
+        results.append({"method": "direct", "error": str(e)})
+
+    # Test FlareSolverr
+    try:
+        async with httpx.AsyncClient(timeout=30) as fs_client:
+            fs_resp = await fs_client.post(
+                settings.flaresolverr_url,
+                json={"cmd": "request.get", "url": url, "maxTimeout": 15000},
+            )
+            if fs_resp.is_success:
+                fs_data = fs_resp.json()
+                fs_html = fs_data.get("solution", {}).get("response", "")
+                fs_status = fs_data.get("solution", {}).get("status", 0)
+                detection = detector.detect(fs_html, fs_status)
+                results.append({"method": "flaresolverr", "status": fs_status, **detection})
+            else:
+                results.append({"method": "flaresolverr", "error": f"HTTP {fs_resp.status_code}"})
+    except (httpx.HTTPError, httpx.RequestError) as e:
+        results.append({"method": "flaresolverr", "error": str(e)})
+
+    return {"success": True, "data": {"url": url, "results": results}}
+
+
+@router.post("/v1/capture/lazy", summary="Detect and handle lazy-loaded content")
+async def detect_lazy_content(
+    url: str = Body(...),
+    auto_scroll: bool = Body(True),
+    max_scrolls: int = Body(5),
+) -> dict[str, Any]:
+    """Detect lazy loading and infinite scroll patterns on a page.
+
+    Optionally generate JS to auto-scroll and load all content.
+    """
+    from lazy_load import (
+        detect_lazy_loading,
+        generate_load_more_script,
+        generate_scroll_script,
+    )
+
+    result = await scraper.scrape(url, {"bypass_cloudflare": True})
+    if result.get("status") != "ok":
+        raise ScrapeError(result.get("error") or "Scrape failed")
+
+    html = result.get("raw_html", "")
+    if not html:
+        client = await get_client()
+        try:
+            resp = await client.get(
+                url, timeout=30, follow_redirects=True, headers={"User-Agent": "Mozilla/5.0"}
+            )
+            html = resp.text
+        except (httpx.HTTPError, httpx.RequestError):
+            raise ScrapeError("Could not fetch raw HTML") from None
+
+    detection = detect_lazy_loading(html)
+    scroll_script = generate_scroll_script(max_scrolls=max_scrolls) if auto_scroll else ""
+    load_more_script = generate_load_more_script() if auto_scroll else ""
+
+    return {
+        "success": True,
+        "data": {
+            "url": url,
+            "detection": detection,
+            "has_lazy_content": any(detection.values()),
+            "scroll_script": scroll_script,
+            "load_more_script": load_more_script,
+        },
+    }
+
+
+# ── Crawl ──
+
+
+@router.post("/v1/crawl", summary="Crawl multiple pages from a URL")
+async def crawl(request: CrawlRequest) -> dict[str, Any]:
+    """Crawl multiple pages from a URL. Supports async webhooks."""
+    if request.webhook:
+        job_id = await queue.create_job("crawl", request.model_dump(), webhook=request.webhook)
+        task = asyncio.create_task(_run_crawl_job(job_id, request))
+        task.add_done_callback(_log_crawl_job_failure)
+        return {"success": True, "data": {"id": job_id, "status": "pending"}}
+
+    pages = await scraper.crawl(
+        request.url,
+        {
+            "max_pages": request.maxPages,
+            "max_depth": request.maxDepth,
+            "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
+        },
+    )
+    return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}}
+
+
+# ── Map ──
+
+
+@router.post("/v1/map", summary="Discover URLs on a site")
+async def map_pages(request: MapRequest) -> dict[str, Any]:
+    """Discover URLs on a site."""
+    urls = await scraper.map_urls(request.url, {"limit": request.limit})
+    return {"success": True, "data": {"links": urls}}
+
+
+# ── Batch ──
+
+
+@router.post("/v1/batch", summary="Scrape multiple URLs in parallel")
+async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[str, Any]:
+    """Scrape multiple URLs in parallel. Firecrawl charges extra for batch."""
+    if len(urls) > 50:
+        raise InvalidRequestError("Max 50 URLs per batch")
+    tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    pages = []
+    for i, r in enumerate(results):
+        if isinstance(r, BaseException):
+            pages.append({"url": urls[i], "error": str(r)})
+        elif isinstance(r, dict):
+            pages.append(
+                {
+                    "url": urls[i],
+                    "markdown": r.get("content", ""),
+                    "method": r.get("method", "unknown"),
+                }
+            )
+    return {"success": True, "data": {"pages": pages, "total": len(pages)}}

routers/templates.py

@@ -0,0 +1,131 @@
+"""Pry — Templates router."""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT.
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Body
+from pydantic import BaseModel
+
+from errors import InvalidRequestError, NotFoundError
+from template_engine import execute_template, get_template, list_templates
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["Templates"])
+
+
+class BatchItem(BaseModel):
+    """A single template execution request within a batch."""
+
+    template_id: str
+    url: str
+    options: dict[str, Any] | None = None
+
+
+class BatchResponse(BaseModel):
+    """Batch execution result."""
+
+    success: bool = True
+    data: dict[str, Any]
+
+
+MAX_BATCH_SIZE = 20
+
+
+@router.get("/v1/templates", summary="List all pre-built scraper templates")
+async def list_templates_endpoint() -> dict[str, Any]:
+    """List all available pre-built scraper templates.
+
+    Templates are one-click extractors for popular websites:
+    Amazon, Walmart, Target, Best Buy, LinkedIn, Indeed, GitHub, etc.
+    """
+    templates = list_templates()
+    categories: dict[str, list[dict[str, Any]]] = {}
+    for t in templates:
+        cat = t.get("category", "general")
+        categories.setdefault(cat, []).append(t)
+    return {
+        "success": True,
+        "data": {"templates": templates, "categories": categories, "total": len(templates)},
+    }
+
+
+@router.get("/v1/templates/{template_id}", summary="Get a scraper template")
+async def get_template_endpoint(template_id: str) -> dict[str, Any]:
+    """Get a specific scraper template with full schema details."""
+    template = get_template(template_id)
+    if not template:
+        raise NotFoundError(f"Template not found: {template_id}")
+    return {"success": True, "data": template}
+
+
+@router.post("/v1/templates/execute", summary="Execute a scraper template against a URL")
+async def execute_template_endpoint(
+    template_id: str = Body(...),
+    url: str = Body(...),
+) -> dict[str, Any]:
+    """Execute a pre-built scraper template against any URL.
+
+    Example: use "amazon_product" template with an Amazon product URL
+    to get structured title, price, rating, description, etc.
+
+    Templates auto-detect the page structure using pre-configured CSS selectors.
+    """
+    result = await execute_template(template_id, url)
+    return result
+
+
+@router.post("/v1/templates/batch", summary="Execute multiple templates/URLs in parallel")
+async def batch_execute(body: list[BatchItem]) -> dict[str, Any]:
+    """Execute multiple scraper templates against multiple URLs in parallel.
+
+    Runs up to 20 template executions concurrently. Each result preserves the
+    original template_id and url for correlation.
+    """
+    if len(body) > MAX_BATCH_SIZE:
+        raise InvalidRequestError(f"Max {MAX_BATCH_SIZE} items per batch, got {len(body)}")
+
+    async def run_item(item: BatchItem) -> dict[str, Any]:
+        try:
+            result = await execute_template(item.template_id, item.url, **(item.options or {}))
+            return {
+                "template_id": item.template_id,
+                "url": item.url,
+                "result": result,
+            }
+        except Exception as e:  # noqa: BLE001
+            return {
+                "template_id": item.template_id,
+                "url": item.url,
+                "error": {"code": "execution_failed", "message": str(e)},
+            }
+
+    results = await asyncio.gather(*[run_item(item) for item in body], return_exceptions=True)
+
+    flat: list[dict[str, Any]] = []
+    failed = 0
+    for r in results:
+        if isinstance(r, Exception):
+            flat.append({"error": {"code": "execution_failed", "message": str(r)}})
+            failed += 1
+        else:
+            if "error" in r:
+                failed += 1
+            flat.append(r)
+
+    return {
+        "success": True,
+        "data": {
+            "results": flat,
+            "total": len(flat),
+            "failed": failed,
+        },
+    }