.actor/actor.json

@@ -0,0 +1,97 @@
+{
+    "actorSpecification": 1,
+    "name": "pry",
+    "title": "Pry — Universal Web Scraper",
+    "description": "Scrape any website with 12-tier automatic fallback. Bypasses Cloudflare, DataDome, Akamai, and other anti-bot systems. Self-hosted, no API keys needed.",
+    "version": "3.0.0",
+    "dockerFile": "./Dockerfile.apify",
+    "input": {
+        "type": "object",
+        "schema": {
+            "type": "object",
+            "title": "Pry Scraper Input",
+            "required": ["urls"],
+            "properties": {
+                "urls": {
+                    "title": "URLs to scrape",
+                    "type": "array",
+                    "description": "One or more URLs to scrape. Each URL is processed through up to 12 fallback tiers.",
+                    "editor": "stringList",
+                    "items": {
+                        "type": "string",
+                        "pattern": "^https?://"
+                    }
+                },
+                "crawl": {
+                    "title": "Crawl linked pages",
+                    "type": "boolean",
+                    "description": "If enabled, follows same-domain links and scrapes them too (up to max_pages total).",
+                    "default": false
+                },
+                "max_pages": {
+                    "title": "Max pages to scrape",
+                    "type": "integer",
+                    "description": "Maximum number of pages to scrape (including crawl).",
+                    "default": 10,
+                    "minimum": 1,
+                    "maximum": 1000
+                },
+                "timeout": {
+                    "title": "Timeout (seconds)",
+                    "type": "integer",
+                    "description": "Per-URL request timeout in seconds.",
+                    "default": 30,
+                    "minimum": 5,
+                    "maximum": 120
+                },
+                "output_format": {
+                    "title": "Output format",
+                    "type": "string",
+                    "description": "The output format for extracted content.",
+                    "enum": ["markdown", "html", "text"],
+                    "default": "markdown"
+                },
+                "proxy_config": {
+                    "title": "Proxy configuration",
+                    "type": "object",
+                    "description": "Proxy settings for scraping.",
+                    "properties": {
+                        "use_apify_proxies": {
+                            "title": "Use Apify proxy",
+                            "type": "boolean",
+                            "description": "Route requests through Apify's datacenter proxy.",
+                            "default": true
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "output": {
+        "type": "object",
+        "schema": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "required": ["url", "status"],
+                "properties": {
+                    "url": { "type": "string" },
+                    "status": { "type": "string", "enum": ["ok", "error", "skipped"] },
+                    "content": { "type": "string" },
+                    "method": { "type": "string" },
+                    "error": { "type": "string" },
+                    "scraped_at": { "type": "string", "format": "date-time" },
+                    "elapsed_seconds": { "type": "number" },
+                    "metadata": {
+                        "type": "object",
+                        "properties": {
+                            "domain": { "type": "string" },
+                            "content_length": { "type": "integer" },
+                            "method": { "type": "string" }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

.commitlintrc.yml

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+rules:
+  body-leading-blank: [2, always]
+  footer-leading-blank: [2, always]
+  header-max-length: [2, always, 100]
+  scope-case: [2, always, lower-case]
+  subject-case: [2, never, [sentence-case, start-case, upper-case]]
+  subject-empty: [2, never]
+  subject-full-stop: [2, never, "."]
+  type-case: [2, always, lower-case]
+  type-empty: [2, never]
+  type-enum:
+    - 2
+    - always
+    - [feat, fix, docs, style, refactor, perf, test, build, ci, chore, ops, security, revert]

.env.example

@@ -69,3 +69,26 @@ PRY_FLARESOLVERR_URL=http://flaresolverr:8191/v1
 
 # ── Redis ──
 # PRY_REDIS_URL=redis://localhost:6379/0
+
+# ── Database ──
+# PRY_DATABASE_URL=postgresql+asyncpg://pry:pry@localhost/pry
+
+# ── Stealth ──
+# PRY_STEALTH_ENABLED=true
+# PRY_RANDOM_USER_AGENT=true
+# PRY_WEBDRIVER_OVERRIDE=true
+# PRY_CANVAS_NOISE=true
+# PRY_WEBRTC_DISABLE=true
+# PRY_GEOLOCATION_SPOOF=true
+# PRY_MIN_DELAY_MS=500
+# PRY_MAX_DELAY_MS=3000
+
+# ── Output ──
+# PRY_DEFAULT_FORMAT=markdown
+# PRY_MAX_CHARS=100000
+# PRY_INCLUDE_LINKS=true
+
+# ── x402 / MCP ──
+# PRY_X402_ENABLED=false
+# PRY_X402_PAY_TO=
+# PRY_MCP_ENABLED=true

.github/workflows/ci.yml

@@ -10,6 +10,13 @@ on:
   pull_request:
     branches: [main]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.12"
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -17,7 +24,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
       - run: pip install ruff
       - run: ruff check .
       - run: ruff format --check .
@@ -28,8 +37,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
-      - run: pip install mypy httpx trafilatura lxml markdownify pydantic pyyaml pandas
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - run: pip install -e ".[dev]"
       - run: mypy --python-version 3.12 --strict --ignore-missing-imports --exclude 'build/|dist/|.git/|__pycache__/' *.py
 
   test:
@@ -38,9 +49,11 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
-      - run: pip install pytest pytest-asyncio pytest-cov httpx trafilatura lxml markdownify pydantic pyyaml pandas
-      - run: pytest tests/ -v --cov=. --cov-report=term-missing
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - run: pip install -e ".[dev]"
+      - run: pytest tests/ -v --cov=. --cov-report=term-missing --cov-fail-under=40
 
   security:
     runs-on: ubuntu-latest
@@ -48,6 +61,29 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
       - run: pip install bandit
       - run: bandit -r . -x tests/,.venv,__pycache__
+
+  gitleaks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: gitleaks/gitleaks-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  commitlint:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: wagoid/commitlint-github-action@v6
+        with:
+          configFile: .commitlintrc.yml

Dockerfile

@@ -49,4 +49,4 @@ HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8002", \
-     "--workers", "2", "--timeout-keep-alive", "120", "--limit-concurrency", "20"]
+     "--workers", "1", "--timeout-keep-alive", "120", "--limit-concurrency", "20"]

Dockerfile.apify

@@ -0,0 +1,25 @@
+# Pry — Apify Actor Dockerfile
+# Lightweight image for Apify platform deployment.
+# Unlike the main Dockerfile, this skips Playwright/Chromium
+# (the Apify platform provides browser rendering via Puppeteer).
+# Only the HTTP/TLS/cloudscraper tiers are available here.
+FROM apify/actor-python:3.12
+
+# Copy everything
+COPY --chmod=755 . /usr/src/app
+
+WORKDIR /usr/src/app
+
+# Install runtime dependencies (no dev deps)
+RUN pip install --no-cache-dir -r requirements.txt \
+    && rm -rf /root/.cache
+
+# Install Apify SDK (not in requirements.txt — only needed for the actor)
+RUN pip install --no-cache-dir "apify~=1.7.0" \
+    && rm -rf /root/.cache
+
+# The actor entry point
+ENV PRY_ACTOR=true
+ENV PYTHONUNBUFFERED=1
+
+ENTRYPOINT ["python3", "-m", "apify_actor"]

account_manager.py

@@ -35,7 +35,7 @@ class AccountPool:
         import uuid
 
         account_id = uuid.uuid4().hex[:12]
-        account = {
+        account: dict[str, Any] = {
             "id": account_id,
             "site": site,
             "credentials": credentials,

advanced.py

@@ -208,7 +208,7 @@ class PryAdvanced:
             "from",
             "they",
         }
-        freq = {}
+        freq: dict[str, int] = {}
         for w in words:
             if w not in stop_words:
                 freq[w] = freq.get(w, 0) + 1

api.py

@@ -25,10 +25,10 @@ from deps import advanced, automator, extractor, queue, ratelimiter, scraper
 from errors import (
     PryError,
 )
-from mcp_production import make_fallback_server, register_all
 from mcp_sse import mcp_post_message, mcp_sse_endpoint
 from observability import REQUEST_COUNT, REQUEST_LATENCY, setup_tracing
 from pipeline import get_pipeline
+from pry_mcp import make_fallback_server, register_all
 from routers.advanced import router as advanced_router
 from routers.agency import router as agency_router
 from routers.ai import router as ai_router

apify_actor.py

@@ -0,0 +1,240 @@
+"""Pry Apify Actor — web scraping + browser automation on the Apify platform.
+
+Runs Pry's multi-tier scraper (UltimateScraper) as an Apify actor.
+Accepts a list of URLs, scrapes each one through up to 12 fallback tiers,
+and pushes the results (markdown content + metadata) to the Apify dataset.
+
+Usage:
+    Deploy to Apify Console or run locally with:
+        apify run -p
+
+Input (JSON):
+    {
+        "urls": ["https://example.com", ...],
+        "max_pages": 1,
+        "timeout": 30,
+        "output_format": "markdown",
+        "proxy_config": {
+            "use_apify_proxies": true,
+            "proxy_country": "US"
+        }
+    }
+
+Output (dataset items):
+    {
+        "url": "https://example.com",
+        "status": "ok" | "error",
+        "content": "...markdown or html...",
+        "method": "direct" | "flaresolverr" | ...,
+        "metadata": {
+            "title": "...",
+            "description": "...",
+            "domain": "example.com",
+            "scraped_at": "2026-07-03T..."
+        }
+    }
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+from datetime import UTC, datetime
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+from errors import InvalidRequestError
+from ultimate_scraper import UltimateScraper
+from url_guard import validate_url
+
+logger = logging.getLogger(__name__)
+
+# Apify SDK is imported lazily inside main() so this file doesn't crash
+# when imported from the main Pry codebase (where apify isn't installed).
+
+
+def _extract_links(html: str, base_url: str) -> list[str]:
+    """Extract same-domain links from raw HTML for crawling."""
+    links: list[str] = []
+    parsed_base = urlparse(base_url)
+    domain = parsed_base.netloc
+    scheme = parsed_base.scheme
+
+    for href in re.findall(r'href=["\'](https?://[^"\']+)["\']', html, re.IGNORECASE):
+        link = urljoin(base_url, href)
+        link_parsed = urlparse(link)
+        if link_parsed.netloc == domain and link_parsed.scheme in ("http", "https"):
+            # Deduplicate by removing fragment
+            clean = link.split("#")[0]
+            if clean not in links:
+                links.append(clean)
+        elif link.startswith("/"):
+            full = f"{scheme}://{domain}{link}"
+            if full not in links:
+                links.append(full)
+
+    return links[:50]  # cap per page
+
+
+async def scrape_url(
+    scraper: UltimateScraper,
+    url: str,
+    options: dict[str, Any],
+) -> dict[str, Any]:
+    """Scrape a single URL and return a structured result."""
+    start = time.time()
+    result = await scraper.scrape(url, options)
+    elapsed = round(time.time() - start, 2)
+
+    status = result.get("status", "error")
+    output: dict[str, Any] = {
+        "url": url,
+        "status": status,
+        "method": result.get("method", "unknown"),
+        "scraped_at": datetime.now(UTC).isoformat(),
+        "elapsed_seconds": elapsed,
+    }
+
+    if status == "ok":
+        output["content"] = result.get("content", "")
+        output["raw_html"] = result.get("raw_html", "")
+        output["metadata"] = {
+            "domain": urlparse(url).netloc,
+            "content_length": len(result.get("content", "") or ""),
+            "method": result.get("method", "unknown"),
+        }
+    else:
+        output["error"] = result.get("error", "unknown_error")
+
+    return output
+
+
+async def main() -> None:
+    """Pry Apify Actor entry point."""
+    # Lazy import so this file is safe to import from the main codebase.
+    from apify import Actor
+
+    async with Actor:
+        actor_input = await Actor.get_input() or {}
+
+        # Parse input
+        raw_urls = actor_input.get("urls", [])
+        if isinstance(raw_urls, str):
+            raw_urls = [raw_urls]
+        if not raw_urls:
+            Actor.log.info("No URLs provided in input; nothing to scrape.")
+            await Actor.push_data({"status": "skipped", "reason": "no_urls"})
+            return
+
+        max_pages = max(1, int(actor_input.get("max_pages", 1)))
+        timeout = int(actor_input.get("timeout", 30))
+        output_format = actor_input.get("output_format", "markdown")
+        crawl = bool(actor_input.get("crawl", False))
+
+        # Build scraper options
+        base_options: dict[str, Any] = {
+            "timeout": timeout,
+        }
+        if actor_input.get("proxy_config", {}).get("use_apify_proxies"):
+            # Apify provides HTTP proxies via http://user:pass@proxy.apify.com:8000
+            proxy_url = Actor.config.get("proxy_url")
+            if proxy_url:
+                base_options["proxy_url"] = proxy_url
+
+        scraper = UltimateScraper()
+        visited: set[str] = set()
+        to_visit: list[str] = list(raw_urls)
+        total_scraped = 0
+        total_errors = 0
+
+        Actor.log.info(
+            "pry_actor_start",
+            extra={
+                "urls": len(raw_urls),
+                "max_pages": max_pages,
+                "crawl": crawl,
+                "timeout": timeout,
+            },
+        )
+
+        while to_visit and total_scraped < max_pages:
+            url = to_visit.pop(0)
+            if url in visited:
+                continue
+            visited.add(url)
+
+            # Validate URL before scraping
+            try:
+                validate_url(url)
+            except InvalidRequestError as e:
+                Actor.log.warning("pry_actor_skip_invalid", extra={"url": url, "error": str(e)})
+                await Actor.push_data(
+                    {
+                        "url": url,
+                        "status": "error",
+                        "error": f"Invalid URL: {e}",
+                        "scraped_at": datetime.now(UTC).isoformat(),
+                    }
+                )
+                total_errors += 1
+                continue
+
+            # Scrape with options
+            Actor.log.info("pry_actor_scrape", extra={"url": url, "remaining": len(to_visit)})
+            options = dict(base_options)
+            if output_format:
+                options["output_format"] = output_format
+
+            output = await scrape_url(scraper, url, options)
+
+            # For crawled URLs, try to extract more links
+            if crawl and output["status"] == "ok":
+                html = output.get("raw_html", "")
+                if html:
+                    discovered = _extract_links(html, url)
+                    new_links = [
+                        link for link in discovered if link not in visited and link not in to_visit
+                    ]
+                    to_visit.extend(new_links[:20])  # cap frontier per page
+                    Actor.log.debug(
+                        "pry_actor_links_discovered",
+                        extra={"url": url, "new_links": len(new_links)},
+                    )
+
+            await Actor.push_data(output)
+            total_scraped += 1
+            if output["status"] != "ok":
+                total_errors += 1
+
+            # Report progress to Apify
+            await Actor.set_value(
+                "STATE",
+                {
+                    "visited": len(visited),
+                    "total_scraped": total_scraped,
+                    "total_errors": total_errors,
+                    "queue_remaining": len(to_visit),
+                },
+            )
+
+        Actor.log.info(
+            "pry_actor_finish",
+            extra={
+                "total_scraped": total_scraped,
+                "total_errors": total_errors,
+                "urls_visited": len(visited),
+            },
+        )
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

cache.py

@@ -12,6 +12,8 @@ import json
 import time
 from collections import OrderedDict
 
+from observability import CACHE_HITS
+
 
 class ResponseCache:
     """Two-tier cache: local LRU + optional Redis persistence.
@@ -44,6 +46,8 @@ class ResponseCache:
             if entry["expires"] > now:
                 self._hits += 1
                 self._cache.move_to_end(key)
+                if CACHE_HITS:
+                    CACHE_HITS.labels(cache_type="local").inc()
                 return entry["data"]
             else:
                 del self._cache[key]

cli.py

@@ -393,7 +393,7 @@ if click is not None:
         sys.path.insert(0, ".")
         import asyncio
 
-        from mcp_production import main
+        from pry_mcp import main
 
         asyncio.run(main())
 

config.py

@@ -1,80 +0,0 @@
-"""PryScraper — global configuration.
-
-Loads settings from environment variables (via Pydantic Settings).
-All secrets come from gopass, never from .env files committed to git.
-"""
-
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 Rug Munch Media LLC
-#
-# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
-# Licensed under MIT. See LICENSE.
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Literal
-
-from pydantic_settings import BaseSettings, SettingsConfigDict
-
-
-class Settings(BaseSettings):
-    """PryScraper application settings."""
-
-    model_config = SettingsConfigDict(
-        env_file=".env",
-        env_file_encoding="utf-8",
-        case_sensitive=False,
-        extra="ignore",
-    )
-
-    # Core
-    app_name: str = "PryScraper"
-    app_version: str = "3.0.0"
-    environment: Literal["dev", "staging", "prod"] = "dev"
-    log_level: str = "INFO"
-    port: int = 8002
-
-    # Database
-    database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry"
-
-    # Cache
-    redis_url: str = "redis://localhost:6379/0"
-
-    # LLM Providers
-    openai_api_key: str = ""
-    anthropic_api_key: str = ""
-    cohere_api_key: str = ""
-    ollama_url: str = "http://localhost:11434"
-
-    # Stealth
-    stealth_enabled: bool = True
-    random_user_agent: bool = True
-    min_delay_ms: int = 500
-    max_delay_ms: int = 3000
-
-    # Rate limiting
-    rate_limit_per_domain: int = 10  # requests per second
-    rate_limit_per_ip: int = 60
-
-    # x402 Payment
-    x402_enabled: bool = False
-    x402_pay_to: str = ""
-
-    # MCP
-    mcp_enabled: bool = True
-    mcp_tools_count: int = 8
-
-    # Storage
-    screenshot_dir: Path = Path("/tmp/pry-screenshots")  # nosec B108
-    cache_ttl_seconds: int = 3600
-
-
-_settings: Settings | None = None
-
-
-def get_settings() -> Settings:
-    """Get cached settings instance."""
-    global _settings
-    if _settings is None:
-        _settings = Settings()
-    return _settings

db.py

@@ -62,8 +62,8 @@ import json
 import logging
 import os
 import sqlite3
-from collections.abc import Iterator
-from contextlib import contextmanager
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager, contextmanager
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
@@ -85,6 +85,7 @@ try:
     )
     from sqlalchemy.engine import Engine
     from sqlalchemy.exc import OperationalError, ProgrammingError, SQLAlchemyError
+    from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
     from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
 
     _HAS_SA = True
@@ -118,9 +119,34 @@ def _resolve_database_url() -> str:
     return DEFAULT_SQLITE_URL
 
 
+def _resolve_async_database_url() -> str:
+    """Resolve the async database URL.
+
+    Converts ``postgresql://...`` to ``postgresql+asyncpg://...`` so the
+    same ``PRY_DATABASE_URL`` env var works for both sync and async uses.
+    SQLite stays as ``sqlite+aiosqlite://...``.
+    """
+    url = os.getenv(DATABASE_URL_ENV)
+    if not url:
+        # SQLite async: use aiosqlite driver
+        return f"sqlite+aiosqlite:///{DEFAULT_DB_PATH}"
+    if url.startswith("postgresql://"):
+        url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
+    elif url.startswith("postgresql+psycopg2://"):
+        url = url.replace("postgresql+psycopg2://", "postgresql+asyncpg://", 1)
+    elif url.startswith("sqlite://"):
+        url = url.replace("sqlite://", "sqlite+aiosqlite://", 1)
+    return url
+
+
 _engine: Engine | None = None
 _SessionLocal: sessionmaker[Session] | None = None
 
+# Async engine/session for use in async FastAPI endpoints.
+# These are lazily initialized alongside the sync engine.
+_async_engine: Any = None  # AsyncEngine (type is Any to avoid import-order issues)
+_async_session_local: Any = None  # async_sessionmaker[AsyncSession]
+
 
 def get_engine() -> Engine:
     """Get the SQLAlchemy engine. Lazily creates one on first call."""
@@ -184,6 +210,61 @@ def session_scope() -> Iterator[Session]:
         s.close()
 
 
+# ── Async engine / session (for async FastAPI endpoints) ──────────
+
+
+def get_async_engine() -> Any:
+    """Get the async SQLAlchemy engine. Lazily creates one on first call."""
+    global _async_engine, _async_session_local
+    if not _HAS_SA:
+        raise RuntimeError("sqlalchemy is not installed; pip install 'sqlalchemy>=2.0'")
+    if _async_engine is None:
+        url = _resolve_async_database_url()
+        connect_args: dict[str, Any] = {}
+        if url.startswith("sqlite"):
+            connect_args["check_same_thread"] = False
+        _async_engine = create_async_engine(url, connect_args=connect_args, echo=False)
+        _async_session_local = async_sessionmaker(
+            bind=_async_engine, autoflush=False, expire_on_commit=False
+        )
+        logger.info(
+            "db_async_engine_initialized",
+            extra={"url": url.split("@")[-1] if "@" in url else url},
+        )
+    return _async_engine
+
+
+async def get_async_session() -> AsyncSession:
+    """Get a new async Session. Caller must close it.
+
+    Prefer ``async with async_session_scope() as s:`` for automatic cleanup.
+    """
+    if _async_session_local is None:
+        get_async_engine()
+    assert _async_session_local is not None
+    return _async_session_local()
+
+
+@asynccontextmanager
+async def async_session_scope() -> AsyncIterator[AsyncSession]:
+    """Async context manager: yields an AsyncSession, commits on success,
+    rolls back on error.
+
+    Usage:
+        async with async_session_scope() as s:
+            result = await s.execute(select(Monitor).where(...))
+    """
+    s = await get_async_session()
+    try:
+        yield s
+        await s.commit()
+    except Exception:
+        await s.rollback()
+        raise
+    finally:
+        await s.close()
+
+
 # ── Model base ────────────────────────────────────────────────────
 
 if _HAS_SA:
@@ -827,3 +908,24 @@ def db_health() -> dict[str, Any]:
         }
     except SQLAlchemyError as e:
         return {"available": False, "error": str(e)[:200]}
+
+
+async def db_health_async() -> dict[str, Any]:
+    """Async health check for the database (for async endpoints)."""
+    if not _HAS_SA:
+        return {"available": False, "reason": "sqlalchemy not installed"}
+    try:
+        engine = get_async_engine()
+        async with engine.connect() as conn:
+            await conn.execute(conn.default_schema_name or "SELECT 1")
+        return {
+            "available": True,
+            "url": _resolve_async_database_url().split("@")[-1]
+            if "@" in _resolve_async_database_url()
+            else _resolve_async_database_url(),
+        }
+    except SQLAlchemyError as e:
+        return {"available": False, "error": str(e)[:200]}
+
+
+# ── end of db.py ──

deps.py

@@ -6,8 +6,6 @@ direct access to shared state.
 
 Usage:
     from deps import scraper, cache, automator, ...
-
-Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
 """
 
 # SPDX-License-Identifier: MIT
@@ -19,18 +17,16 @@ from automator import PryAutomator
 from cache import ResponseCache
 from extractor import SchemaExtractor
 from jobqueue import JobQueue
-from mconfig import PryConfig
 from parser import DocumentParser
 from ratelimit import RateLimiter
 from scraper import PryScraper
-
-config = PryConfig()
+from settings import settings
 
 scraper = PryScraper()
 automator = PryAutomator()
 parser = DocumentParser()
 extractor = SchemaExtractor()
 cache = ResponseCache(capacity=1000)
-ratelimiter = RateLimiter(default_rpm=120, burst=200)
+ratelimiter = RateLimiter(default_rpm=settings.rate_limit_rpm, burst=200)
 queue = JobQueue()
 advanced = PryAdvanced(cache=cache)

extractor.py

@@ -14,6 +14,8 @@ import json
 import re
 from typing import Any
 
+from resilience import registry
+
 
 class SchemaExtractor:
     """Extract structured JSON data from scraped markdown content.
@@ -49,6 +51,7 @@ class SchemaExtractor:
                 merged = {**pattern_result, **llm_result}
                 return {k: v for k, v in merged.items() if v is not None and v != ""}
         except Exception:  # noqa: BLE001
+            registry.record_service_result("ollama", False)
             pass
 
         return pattern_result
@@ -106,6 +109,8 @@ class SchemaExtractor:
 
     async def _llm_extract(self, content: str, schema: dict[str, Any]) -> dict[str, Any]:
         """LLM-guided extraction. Returns dict on success, {"_error": msg} on failure."""
+        if not registry.is_service_allowed("ollama"):
+            return {"_error": "ollama circuit breaker open"}
         import httpx
 
         schema_str = json.dumps(schema, indent=2)
@@ -136,7 +141,10 @@ class SchemaExtractor:
             if json_match:
                 obj = json.loads(json_match.group(0))
                 if isinstance(obj, dict):
+                    registry.record_service_result("ollama", True)
                     return obj
+            registry.record_service_result("ollama", True)
             return {"_raw": response[:500]}
         except (json.JSONDecodeError, ValueError) as e:
+            registry.record_service_result("ollama", False)
             return {"_error": str(e)}

mconfig.py

@@ -1,133 +0,0 @@
-"""Pry Config — runtime configuration for proxy, Tor, VPN, and anti-detection.
-Users can configure at startup (env vars) or at runtime (API calls)."""
-
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 Rug Munch Media LLC
-#
-# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
-# Licensed under MIT. See LICENSE.
-
-import json
-import os
-
-CONFIG_FILE = "/app/config.json"
-DEFAULT_CONFIG = {
-    "proxy": {"enabled": False, "type": "", "url": "", "username": "", "password": ""},
-    "tor": {"enabled": False, "socks5_host": "tor", "socks5_port": 9050, "control_port": 9051},
-    "rotation": {
-        "user_agent": "rotate",
-        "ip": "off",
-        "timing": "random",
-        "delay_range": [0.5, 3.0],
-    },
-    "stealth": {
-        "webdriver_override": True,
-        "canvas_noise": True,
-        "webrtc_disable": True,
-        "geolocation_spoof": True,
-    },
-    "retry": {"max_attempts": 3, "min_quality": 20, "backoff": "exponential"},
-    "output": {"default_format": "markdown", "max_chars": 100000, "include_links": True},
-    "rate_limit": {"rpm": 120, "burst": 200},
-}
-
-
-class PryConfig:
-    """Configuration manager — loads from env vars, config file, and API overrides."""
-
-    def __init__(self):
-        self.config = json.loads(json.dumps(DEFAULT_CONFIG))  # Deep copy
-        self._load_env()
-        self._load_file()
-        self._resolve_proxy_chain()
-
-    def _load_env(self):
-        """Environment variables override defaults."""
-        if os.getenv("TOR_ENABLED", "").lower() in ("1", "true", "yes"):
-            self.config["tor"]["enabled"] = True
-        if os.getenv("TOR_SOCKS5_HOST"):
-            self.config["tor"]["socks5_host"] = os.getenv("TOR_SOCKS5_HOST")
-        if os.getenv("TOR_SOCKS5_PORT"):
-            self.config["tor"]["socks5_port"] = int(os.getenv("TOR_SOCKS5_PORT"))
-        if os.getenv("PROXY_URL"):
-            self.config["proxy"]["enabled"] = True
-            self.config["proxy"]["url"] = os.getenv("PROXY_URL")
-            self.config["proxy"]["type"] = os.getenv("PROXY_TYPE", "http")
-        if os.getenv("PROXY_USERNAME"):
-            self.config["proxy"]["username"] = os.getenv("PROXY_USERNAME")
-        if os.getenv("PROXY_PASSWORD"):
-            self.config["proxy"]["password"] = os.getenv("PROXY_PASSWORD")
-        if os.getenv("IP_ROTATION"):
-            self.config["rotation"]["ip"] = os.getenv("IP_ROTATION")
-        if os.getenv("MAX_RETRIES"):
-            self.config["retry"]["max_attempts"] = int(os.getenv("MAX_RETRIES"))
-        if os.getenv("MIN_QUALITY"):
-            self.config["retry"]["min_quality"] = int(os.getenv("MIN_QUALITY"))
-        if os.getenv("RATE_LIMIT_RPM"):
-            self.config["rate_limit"]["rpm"] = int(os.getenv("RATE_LIMIT_RPM"))
-
-    def _load_file(self):
-        if os.path.exists(CONFIG_FILE):
-            try:
-                with open(CONFIG_FILE) as f:
-                    user_config = json.load(f)
-                self._deep_merge(self.config, user_config)
-            except OSError:
-                pass
-
-    def _deep_merge(self, base: dict, override: dict):
-        for key, value in override.items():
-            if key in base and isinstance(base[key], dict) and isinstance(value, dict):
-                self._deep_merge(base[key], value)
-            else:
-                base[key] = value
-
-    def _resolve_proxy_chain(self):
-        """Build the proxy chain: Tor -> User Proxy -> Direct"""
-        proxies = []
-        if self.config["tor"]["enabled"]:
-            proxies.append(
-                f"socks5://{self.config['tor']['socks5_host']}:{self.config['tor']['socks5_port']}"
-            )
-        if self.config["proxy"]["enabled"]:
-            url = self.config["proxy"]["url"]
-            if self.config["proxy"]["username"]:
-                netloc = url.split("://")[1] if "://" in url else url
-                proxies.append(
-                    f"{self.config['proxy']['type']}://{self.config['proxy']['username']}:{self.config['proxy']['password']}@{netloc}"
-                )
-            else:
-                proxies.append(url)
-        self.config["_proxy_chain"] = proxies
-        self.config["_proxy_url"] = proxies[0] if proxies else None
-
-    def get_proxy_url(self) -> str | None:
-        return self.config.get("_proxy_url")
-
-    def get_proxy_chain(self) -> list:
-        return self.config.get("_proxy_chain", [])
-
-    def get(self, key: str, default=None):
-        keys = key.split(".")
-        val = self.config
-        for k in keys:
-            if isinstance(val, dict):
-                val = val.get(k)
-            else:
-                return default
-            if val is None:
-                return default
-        return val if val is not None else default
-
-    def to_dict(self) -> dict:
-        return {k: v for k, v in self.config.items() if not k.startswith("_")}
-
-    def update(self, updates: dict) -> dict:
-        self._deep_merge(self.config, updates)
-        self._resolve_proxy_chain()
-        try:
-            with open(CONFIG_FILE, "w") as f:
-                json.dump(self.to_dict(), f, indent=2)
-        except OSError:
-            pass
-        return {"status": "ok", "config": self.to_dict()}

mcp_sse.py

@@ -29,7 +29,7 @@ from typing import Any
 from fastapi import HTTPException, Request, Response
 from fastapi.responses import StreamingResponse
 
-from mcp_production import (
+from pry_mcp import (
     register_mcp_notification_observer,
     unregister_mcp_notification_observer,
 )

observability.py

@@ -6,6 +6,7 @@
 """Pry — Observability: Prometheus metrics, OpenTelemetry tracing, structured logging."""
 
 import logging
+import os
 import time
 from contextlib import contextmanager
 
@@ -25,9 +26,19 @@ try:
     from opentelemetry.sdk.trace import TracerProvider
     from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor
 
+    # OTLP exporter is optional (install opentelemetry-exporter-otlp-proto-http)
+    try:
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+        _has_otlp = True
+    except ImportError:
+        _has_otlp = False
+
     _has_otel = True
 except ImportError:
     _has_otel = False
+    _has_otlp = False
 
 
 # Metrics
@@ -49,15 +60,26 @@ else:
 
 
 def setup_tracing(service_name: str = "pry") -> None:
-    """Initialize OpenTelemetry tracing."""
+    """Initialize OpenTelemetry tracing.
+
+    Uses OTLP HTTP exporter if ``OTEL_EXPORTER_OTLP_ENDPOINT`` is set
+    and the OTLP package is installed. Falls back to console export.
+    """
     if not _has_otel:
         return
     try:
         provider = TracerProvider()
-        processor = SimpleSpanProcessor(ConsoleSpanExporter())
+        otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+        if otlp_endpoint and _has_otlp:
+            exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
+            processor = BatchSpanProcessor(exporter)
+            logger.info("tracing_otlp", extra={"endpoint": otlp_endpoint})
+        else:
+            exporter = ConsoleSpanExporter()
+            processor = SimpleSpanProcessor(exporter)
+            logger.info("tracing_console", extra={"service": service_name})
         provider.add_span_processor(processor)
         trace.set_tracer_provider(provider)
-        logger.info("tracing_initialized", extra={"service": service_name})
     except Exception as e:  # noqa: BLE001
         logger.warning("tracing_init_failed", extra={"error": str(e)[:80]})
 

pry_mcp/__init__.py

@@ -0,0 +1,53 @@
+"""Pry - MCP server sub-package.
+
+Split from mcp_production.py for maintainability.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+from pry_mcp.constants import (
+    DEFAULT_BASE_URL,
+    MCP_PROTOCOL_VERSION,
+    SERVER_NAME,
+    SERVER_VERSION,
+)
+from pry_mcp.fallback_server import make_fallback_server
+from pry_mcp.notifications import (
+    MCPLoggingHandler,
+    notify_resource_changed,
+    notify_resource_list_changed,
+    notify_tool_list_changed,
+    register_mcp_notification_observer,
+    set_active_mcp_server,
+    unregister_mcp_notification_observer,
+)
+from pry_mcp.resources import PRY_PROMPTS, PRY_RESOURCES
+from pry_mcp.sdk_server import create_server, main, register_all
+from pry_mcp.tools import PRY_TOOLS
+
+__all__ = [
+    "DEFAULT_BASE_URL",
+    "MCP_PROTOCOL_VERSION",
+    "PRY_PROMPTS",
+    "PRY_RESOURCES",
+    "PRY_TOOLS",
+    "SERVER_NAME",
+    "SERVER_VERSION",
+    "MCPLoggingHandler",
+    "create_server",
+    "main",
+    "make_fallback_server",
+    "notify_resource_changed",
+    "notify_resource_list_changed",
+    "notify_tool_list_changed",
+    "register_all",
+    "register_mcp_notification_observer",
+    "set_active_mcp_server",
+    "unregister_mcp_notification_observer",
+]

pry_mcp/constants.py

@@ -0,0 +1,18 @@
+"""Pry - MCP server constants."""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+import os
+
+# MCP protocol version (per the official spec)
+MCP_PROTOCOL_VERSION = "2024-11-05"
+SERVER_NAME = "pry"
+SERVER_VERSION = "3.0.0"
+
+DEFAULT_BASE_URL = os.getenv("PRY_API_URL", "http://localhost:8002")

pry_mcp/fallback_server.py

@@ -0,0 +1,555 @@
+"""Pry - MCP fallback server implementation.
+
+Implements the MCP protocol when the official MCP SDK is not available.
+Contains FallbackMCPServer and make_fallback_server().
+
+Split from mcp_production.py.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import sys
+from collections.abc import Callable, Coroutine
+from typing import Any
+
+import httpx
+
+from pry_mcp.constants import DEFAULT_BASE_URL, MCP_PROTOCOL_VERSION, SERVER_NAME, SERVER_VERSION
+from pry_mcp.notifications import (
+    _attach_mcp_logging,
+    set_active_mcp_server,
+)
+
+logger = logging.getLogger(__name__)
+
+def make_fallback_server(
+    base_url: str = DEFAULT_BASE_URL,
+    tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]]
+    | None = None,
+) -> Any:
+    """Build a minimal stdio JSON-RPC server if official MCP SDK not available.
+    This implements the MCP 2024-11-05 protocol with all required methods."""
+
+    class FallbackMCPServer:
+        def __init__(
+            self,
+            base_url: str = DEFAULT_BASE_URL,
+            tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]]
+            | None = None,
+        ) -> None:
+            self.tools: dict[str, dict[str, Any]] = {}
+            self.resources: dict[str, dict[str, Any]] = {}
+            self.prompts: dict[str, dict[str, Any]] = {}
+            self._initialized = False
+            self._client_info: dict[str, Any] = {}
+            self.base_url = base_url.rstrip("/")
+            self.tool_executor = tool_executor
+
+            # MCP logging state
+            self._mcp_log_enabled = False
+            self._mcp_log_level = logging.INFO
+
+            # Resource subscription state: uri -> set of subscriber session/client ids
+            self._resource_subscribers: dict[str, set[str]] = {}
+
+        def register_tool(self, name: str, definition: dict[str, Any]) -> None:
+            self.tools[name] = definition
+
+        def register_resource(self, uri: str, definition: dict[str, Any]) -> None:
+            self.resources[uri] = definition
+
+        def register_prompt(self, name: str, definition: dict[str, Any]) -> None:
+            self.prompts[name] = definition
+
+        def list_tools(self) -> dict[str, Any]:
+            return {"tools": list(self.tools.values())}
+
+        def subscribe_resource(self, uri: str, subscriber_id: str = "default") -> bool:
+            """Subscribe a client to updates for a resource URI."""
+            if uri not in self.resources:
+                return False
+            self._resource_subscribers.setdefault(uri, set()).add(subscriber_id)
+            return True
+
+        def unsubscribe_resource(self, uri: str, subscriber_id: str = "default") -> bool:
+            """Unsubscribe a client from updates for a resource URI."""
+            subscribers = self._resource_subscribers.get(uri)
+            if subscribers is None:
+                return False
+            subscribers.discard(subscriber_id)
+            if not subscribers:
+                del self._resource_subscribers[uri]
+            return True
+
+        def set_mcp_log_level(self, level_name: str) -> None:
+            """Set the MCP log level (debug/info/warning/error/critical)."""
+            level_map = {
+                "debug": logging.DEBUG,
+                "info": logging.INFO,
+                "warning": logging.WARNING,
+                "error": logging.ERROR,
+                "critical": logging.CRITICAL,
+            }
+            self._mcp_log_level = level_map.get(level_name.lower(), logging.INFO)
+            self._mcp_log_enabled = True
+
+        def disable_mcp_log(self) -> None:
+            """Disable MCP log forwarding."""
+            self._mcp_log_enabled = False
+
+        async def call_tool(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+            """Execute a tool. Returns proper MCP-compliant response."""
+            if name not in self.tools:
+                return {"error": {"code": -32602, "message": f"Unknown tool: {name}"}}
+
+            # If a local executor is provided (e.g., when mounted inside api.py), use it.
+            if self.tool_executor is not None:
+                try:
+                    result = await self.tool_executor(name, arguments)
+                    return self._tool_result_to_content(result, name)
+                except Exception as e:  # noqa: BLE001
+                    logger.warning(
+                        "mcp_tool_executor_failed", extra={"tool": name, "error": str(e)}
+                    )
+                    return {
+                        "content": [{"type": "text", "text": f"Tool {name} failed: {e}"}],
+                        "isError": True,
+                    }
+
+            tool_map = {
+                "pry_scrape": ("POST", "/v1/scrape"),
+                "pry_crawl": ("POST", "/v1/crawl"),
+                "pry_extract": ("POST", "/v1/extract/css"),
+                "pry_template": ("POST", "/v1/templates/execute"),
+                "pry_search_templates": ("GET", "/v1/templates"),
+                "pry_monitor": ("POST", "/v1/monitor"),
+                "pry_compliance": ("POST", "/v1/compliance/check"),
+                "pry_enrich": ("POST", "/v1/enrich"),
+                "pry_parse_document": ("POST", "/v1/parse"),
+                "pry_screenshot": ("POST", "/v1/screenshot"),
+                "pry_x402_pricing": ("GET", "/v1/x402/pricing"),
+                "pry_referrals": ("GET", "/v1/referrals/catalog"),
+            }
+            if name not in tool_map:
+                return {
+                    "content": [{"type": "text", "text": f"Tool {name} not yet wired to API"}],
+                    "isError": False,
+                }
+            method, path = tool_map[name]
+
+            # Try to call the Pry API directly.
+            try:
+                async with httpx.AsyncClient(timeout=60) as client:
+                    if method == "GET":
+                        response = await client.get(f"{self.base_url}{path}", params=arguments)
+                    else:
+                        response = await client.post(f"{self.base_url}{path}", json=arguments)
+                    response.raise_for_status()
+                    data = response.json()
+                    return self._tool_result_to_content(data, name)
+            except (httpx.HTTPError, httpx.RequestError) as e:
+                logger.warning("mcp_tool_api_call_failed", extra={"tool": name, "error": str(e)})
+                return {
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": (
+                                f"Would call {method} {path} with {arguments}. "
+                                f"Live execution unavailable: {e}"
+                            ),
+                        }
+                    ],
+                    "isError": False,
+                }
+
+        def _tool_result_to_content(self, result: Any, tool_name: str) -> dict[str, Any]:
+            """Normalize a tool result into MCP content array."""
+            if isinstance(result, dict):
+                if "content" in result and isinstance(result["content"], list):
+                    return result
+                text = json.dumps(result, indent=2)
+            elif isinstance(result, list):
+                text = json.dumps(result, indent=2)
+            else:
+                text = str(result)
+            return {
+                "content": [{"type": "text", "text": text}],
+                "isError": False,
+            }
+
+        def read_resource(self, uri: str) -> dict[str, Any]:
+            """Read a resource. Returns proper MCP-compliant response."""
+            if uri not in self.resources:
+                return {"error": {"code": -32602, "message": f"Resource not found: {uri}"}}
+            res = self.resources[uri]
+            text = self._resource_text(uri)
+            return {
+                "contents": [
+                    {
+                        "uri": uri,
+                        "mimeType": res.get("mimeType", "text/plain"),
+                        "text": text,
+                    }
+                ],
+            }
+
+        def _resource_text(self, uri: str) -> str:
+            """Generate resource contents without external I/O."""
+            if uri == "pry://catalog":
+                try:
+                    from template_engine import list_templates
+
+                    templates = list_templates()
+                    categories: dict[str, list[str]] = {}
+                    for t in templates:
+                        cat = t.get("category", "general")
+                        categories.setdefault(cat, []).append(t.get("template_id", "unknown"))
+                    return json.dumps(
+                        {
+                            "total_templates": len(templates),
+                            "categories": categories,
+                            "note": "Use pry_search_templates to query dynamically.",
+                        },
+                        indent=2,
+                    )
+                except (json.JSONDecodeError, ValueError) as e:
+                    return json.dumps({"error": f"Could not load template catalog: {e}"})
+            if uri == "pry://stats":
+                return json.dumps(
+                    {
+                        "server": SERVER_NAME,
+                        "version": SERVER_VERSION,
+                        "protocol_version": MCP_PROTOCOL_VERSION,
+                        "tools_count": len(self.tools),
+                        "resources_count": len(self.resources),
+                        "prompts_count": len(self.prompts),
+                    },
+                    indent=2,
+                )
+            if uri == "pry://x402/pricing":
+                try:
+                    from x402 import X402Handler
+
+                    return json.dumps(X402Handler().get_stats(), indent=2)
+                except (json.JSONDecodeError, ValueError) as e:
+                    return json.dumps({"error": f"Could not load x402 pricing: {e}"})
+            if uri == "pry://referrals":
+                try:
+                    from referrals import ReferralTracker
+
+                    return json.dumps(ReferralTracker().get_catalog(), indent=2)  # type: ignore[no-untyped-call]
+                except (json.JSONDecodeError, ValueError) as e:
+                    return json.dumps({"error": f"Could not load referrals: {e}"})
+            return f"Resource {uri} (placeholder)"
+
+        def get_prompt(self, name: str, arguments: dict[str, Any]) -> dict[str, Any]:
+            """Get a prompt. Returns proper MCP-compliant response."""
+            if name not in self.prompts:
+                return {"error": {"code": -32602, "message": f"Prompt not found: {name}"}}
+            prompt = self.prompts[name]
+            messages = self._prompt_messages(name, arguments)
+            return {
+                "description": prompt.get("description", ""),
+                "messages": messages,
+            }
+
+        def _prompt_messages(self, name: str, arguments: dict[str, Any]) -> list[dict[str, Any]]:
+            """Build real prompt messages for named prompts."""
+            if name == "research_company":
+                company = arguments.get("company_name", "the company")
+                website = arguments.get("website", "")
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Research {company}. "
+                                f"{'Start by scraping ' + website + '. ' if website else ''}"
+                                "Use pry_scrape and pry_enrich to gather data, then summarize: "
+                                "what they do, target market, key products, competitive positioning, "
+                                "and any risks or opportunities."
+                            ),
+                        },
+                    }
+                ]
+            if name == "compare_products":
+                query = arguments.get("product_query", "the product")
+                sites = arguments.get("sites", [])
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Compare prices and details for '{query}' across {', '.join(sites)}. "
+                                "Use pry_template with amazon-product, walmart-product, etc. "
+                                "Return a side-by-side comparison table."
+                            ),
+                        },
+                    }
+                ]
+            if name == "extract_contacts":
+                url = arguments.get("url", "")
+                max_pages = arguments.get("max_pages", 20)
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Extract all contact information from {url}. "
+                                f"Crawl up to {max_pages} pages. "
+                                "Use pry_crawl and regex to find emails, phones, and social profiles."
+                            ),
+                        },
+                    }
+                ]
+            if name == "monitor_competitor":
+                url = arguments.get("url", "")
+                what = arguments.get("what_to_track", "changes")
+                webhook = arguments.get("webhook_url", "")
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Set up monitoring for {url}. Track {what}. "
+                                f"{'Notify via webhook: ' + webhook + '. ' if webhook else ''}"
+                                "Use pry_monitor."
+                            ),
+                        },
+                    }
+                ]
+            if name == "analyze_article":
+                url = arguments.get("url", "")
+                focus = arguments.get("focus", "")
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Scrape and analyze this article: {url}. "
+                                f"{'Focus on: ' + focus + '. ' if focus else ''}"
+                                "Summarize key points, sentiment, entities, and claims."
+                            ),
+                        },
+                    }
+                ]
+            if name == "extract_table":
+                url = arguments.get("url", "")
+                desc = arguments.get("table_description", "")
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Extract the data table from {url}. Table: {desc}. "
+                                "Use pry_extract with CSS selectors or pry_scrape + structured output."
+                            ),
+                        },
+                    }
+                ]
+            if name == "scrape_with_schema":
+                url = arguments.get("url", "")
+                fields = arguments.get("fields", "")
+                return [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": (
+                                f"Scrape {url} and extract these fields: {fields}. "
+                                "Use pry_extract with a custom JSON schema."
+                            ),
+                        },
+                    }
+                ]
+            arg_text = ", ".join(f"{k}={v}" for k, v in arguments.items())
+            return [
+                {
+                    "role": "user",
+                    "content": {
+                        "type": "text",
+                        "text": f"Execute the {self.prompts[name].get('title', name)} prompt with arguments: {arg_text}",
+                    },
+                }
+            ]
+
+        def complete(self, ref: dict[str, Any], argument: dict[str, Any]) -> dict[str, Any]:
+            """Provide argument completion suggestions."""
+            ref_type = ref.get("type", "")
+            ref_name = ref.get("name", "") or ref.get("uri", "")
+            arg_name = argument.get("name", "")
+            arg_value = argument.get("value", "")
+            values: list[str] = []
+            if (
+                ref_type == "ref/prompt"
+                and ref_name == "extract_contacts"
+                and arg_name == "max_pages"
+            ):
+                values = ["10", "20", "50", "100"]
+            elif ref_type == "ref/tool" and arg_name == "template_id":
+                try:
+                    from template_engine import list_templates
+
+                    templates = list_templates()
+                    values = [
+                        t["template_id"]
+                        for t in templates
+                        if arg_value.lower() in t.get("template_id", "").lower()
+                    ][:10]
+                except Exception:  # noqa: BLE001
+                    values = []
+            elif ref_type == "ref/tool" and arg_name == "url":
+                values = []
+            elif arg_name in ("format", "mime_type"):
+                values = ["json", "csv", "markdown", "html"]
+            return {"values": values}
+
+        async def handle_request(self, request: dict[str, Any]) -> dict[str, Any] | None:
+            """Handle a JSON-RPC 2.0 request. Implements MCP 2024-11-05."""
+            method = request.get("method", "")
+            req_id = request.get("id", 0)
+            params = request.get("params", {})
+
+            try:
+                if method == "initialize":
+                    self._initialized = True
+                    self._client_info = params.get("clientInfo", {})
+                    _attach_mcp_logging(self)
+                    set_active_mcp_server(self)
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {
+                            "protocolVersion": MCP_PROTOCOL_VERSION,
+                            "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION},
+                            "capabilities": {
+                                "tools": {"listChanged": True},
+                                "resources": {"subscribe": True, "listChanged": True},
+                                "prompts": {"listChanged": True},
+                                "logging": {},
+                            },
+                        },
+                    }
+                if method == "notifications/initialized":
+                    return None
+                if method == "ping":
+                    return {"jsonrpc": "2.0", "id": req_id, "result": {}}
+                if method == "tools/list":
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {"tools": list(self.tools.values())},
+                    }
+                if method == "tools/call":
+                    tool_result = await self.call_tool(
+                        params.get("name", ""), params.get("arguments", {})
+                    )
+                    if "error" in tool_result:
+                        return {
+                            "jsonrpc": "2.0",
+                            "id": req_id,
+                            "error": tool_result["error"],
+                        }
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {
+                            "content": tool_result.get("content", []),
+                            "isError": tool_result.get("isError", False),
+                        },
+                    }
+                if method == "resources/list":
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {"resources": list(self.resources.values())},
+                    }
+                if method == "resources/read":
+                    read_result = self.read_resource(params.get("uri", ""))
+                    if "error" in read_result:
+                        return {"jsonrpc": "2.0", "id": req_id, "error": read_result["error"]}
+                    return {"jsonrpc": "2.0", "id": req_id, "result": read_result}
+                if method == "resources/subscribe":
+                    uri = params.get("uri", "")
+                    ok = self.subscribe_resource(uri)
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {"subscribed": ok},
+                    }
+                if method == "resources/unsubscribe":
+                    uri = params.get("uri", "")
+                    ok = self.unsubscribe_resource(uri)
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {"unsubscribed": ok},
+                    }
+                if method == "prompts/list":
+                    return {
+                        "jsonrpc": "2.0",
+                        "id": req_id,
+                        "result": {"prompts": list(self.prompts.values())},
+                    }
+                if method == "prompts/get":
+                    prompt_result = self.get_prompt(
+                        params.get("name", ""), params.get("arguments", {})
+                    )
+                    if "error" in prompt_result:
+                        return {"jsonrpc": "2.0", "id": req_id, "error": prompt_result["error"]}
+                    return {"jsonrpc": "2.0", "id": req_id, "result": prompt_result}
+                if method == "completion/complete":
+                    completion = self.complete(params.get("ref", {}), params.get("argument", {}))
+                    return {"jsonrpc": "2.0", "id": req_id, "result": completion}
+                if method == "logging/setLevel":
+                    level = params.get("level", "info")
+                    self.set_mcp_log_level(level)
+                    return {"jsonrpc": "2.0", "id": req_id, "result": {}}
+                return {
+                    "jsonrpc": "2.0",
+                    "id": req_id,
+                    "error": {"code": -32601, "message": f"Method not found: {method}"},
+                }
+            except Exception as e:  # noqa: BLE001
+                return {
+                    "jsonrpc": "2.0",
+                    "id": req_id,
+                    "error": {"code": -32603, "message": f"Internal error: {e}"},
+                }
+
+        async def run_stdio(self) -> None:
+            """Run the server over stdio (JSON-RPC 2.0)."""
+            while True:
+                try:
+                    line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline)
+                    if not line:
+                        break
+                    line = line.strip()
+                    if not line:
+                        continue
+                    request = json.loads(line)
+                    response = await self.handle_request(request)
+                    if response is not None:
+                        sys.stdout.write(json.dumps(response) + "\n")
+                        sys.stdout.flush()
+                except json.JSONDecodeError:
+                    continue
+                except (EOFError, KeyboardInterrupt):
+                    break
+
+    return FallbackMCPServer()

pry_mcp/notifications.py

@@ -0,0 +1,135 @@
+"""Pry - MCP notification system.
+
+Observer pattern for outgoing MCP notifications, MCP logging handler,
+and notification helper functions.
+
+Split from mcp_production.py.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any, ClassVar
+
+logger = logging.getLogger(__name__)
+
+# Observer callbacks for outgoing MCP notifications (used by SSE transport).
+# Each callback receives a JSON-RPC notification dict.
+NotificationObserver = Callable[[dict[str, Any]], None]
+_mcp_notification_observers: list[NotificationObserver] = []
+
+
+def register_mcp_notification_observer(observer: NotificationObserver) -> None:
+    """Register a callback that receives every outgoing MCP notification."""
+    if observer not in _mcp_notification_observers:
+        _mcp_notification_observers.append(observer)
+
+
+def unregister_mcp_notification_observer(observer: NotificationObserver) -> None:
+    """Remove a notification observer."""
+    if observer in _mcp_notification_observers:
+        _mcp_notification_observers.remove(observer)
+
+
+def _send_mcp_notification(notification: dict[str, Any]) -> None:
+    """Dispatch an MCP notification to all registered observers."""
+    for observer in list(_mcp_notification_observers):
+        try:
+            observer(notification)
+        except Exception as e:  # noqa: BLE001
+            logger.warning("mcp_notification_observer_failed", extra={"error": str(e)})
+
+
+class MCPLoggingHandler(logging.Handler):
+    """Python logging handler that forwards log records as MCP notifications."""
+
+    _LEVEL_MAP: ClassVar[dict[int, str]] = {
+        logging.DEBUG: "debug",
+        logging.INFO: "info",
+        logging.WARNING: "warning",
+        logging.ERROR: "error",
+        logging.CRITICAL: "critical",
+    }
+
+    def __init__(self, server: Any) -> None:
+        super().__init__()
+        self.server = server
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if not getattr(self.server, "_mcp_log_enabled", False):
+            return
+        min_level = getattr(self.server, "_mcp_log_level", logging.INFO)
+        if record.levelno < min_level:
+            return
+        notification = {
+            "jsonrpc": "2.0",
+            "method": "notifications/message",
+            "params": {
+                "level": self._LEVEL_MAP.get(record.levelno, "info"),
+                "logger": record.name,
+                "data": self.format(record),
+            },
+        }
+        _send_mcp_notification(notification)
+
+
+# Attach one global handler to the pry logger tree once a server is created.
+_mcp_log_handler: MCPLoggingHandler | None = None
+
+
+def _attach_mcp_logging(server: Any) -> None:
+    """Attach the MCP forwarding log handler to the pry logger."""
+    global _mcp_log_handler
+    if _mcp_log_handler is None:
+        _mcp_log_handler = MCPLoggingHandler(server)
+        _mcp_log_handler.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
+        logging.getLogger("pry").addHandler(_mcp_log_handler)
+        logging.getLogger("pry").setLevel(logging.DEBUG)
+
+
+# Global server instance used by notification helpers.
+_active_mcp_server: Any | None = None
+
+
+def set_active_mcp_server(server: Any) -> None:
+    """Mark an MCP server instance as active so helpers can use it."""
+    global _active_mcp_server
+    _active_mcp_server = server
+
+
+def notify_resource_changed(uri: str) -> None:
+    """Notify all connected MCP clients that a resource has changed."""
+    _send_mcp_notification(
+        {
+            "jsonrpc": "2.0",
+            "method": "notifications/resources/updated",
+            "params": {"uri": uri},
+        }
+    )
+
+
+def notify_resource_list_changed() -> None:
+    """Notify all connected MCP clients that the resource list changed."""
+    _send_mcp_notification(
+        {
+            "jsonrpc": "2.0",
+            "method": "notifications/resources/list_changed",
+        }
+    )
+
+
+def notify_tool_list_changed() -> None:
+    """Notify all connected MCP clients that the tool list changed."""
+    _send_mcp_notification(
+        {
+            "jsonrpc": "2.0",
+            "method": "notifications/tools/list_changed",
+        }
+    )

pry_mcp/resources.py

@@ -0,0 +1,164 @@
+"""Pry - MCP resource and prompt definitions.
+
+PRY_RESOURCES and PRY_PROMPTS lists for the MCP server.
+
+Split from mcp_production.py.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+# ── Resource Definitions (per MCP spec) ──
+
+PRY_RESOURCES = [
+    {
+        "uri": "pry://catalog",
+        "name": "Scraper Template Catalog",
+        "description": "All 110+ available scraper templates organized by category. Use pry_search_templates tool for dynamic queries.",
+        "mimeType": "application/json",
+        "annotations": {
+            "audience": ["user", "assistant"],
+            "priority": 0.9,
+        },
+    },
+    {
+        "uri": "pry://stats",
+        "name": "Pry Usage Statistics",
+        "description": "Current Pry server statistics: uptime, requests served, templates used, costs.",
+        "mimeType": "application/json",
+        "annotations": {
+            "audience": ["assistant"],
+            "priority": 0.5,
+        },
+    },
+    {
+        "uri": "pry://x402/pricing",
+        "name": "x402 Pay-per-call Pricing",
+        "description": "x402 pricing for all paid operations. Use to display cost to users before they make a paid call.",
+        "mimeType": "application/json",
+        "annotations": {
+            "audience": ["user", "assistant"],
+            "priority": 0.8,
+        },
+    },
+    {
+        "uri": "pry://referrals",
+        "name": "Referral Catalog",
+        "description": "Pry's referral program catalog. 60+ providers with affiliate links for revenue sharing.",
+        "mimeType": "application/json",
+        "annotations": {
+            "audience": ["assistant"],
+            "priority": 0.3,
+        },
+    },
+]
+
+# ── Prompt Definitions (per MCP spec) ──
+
+PRY_PROMPTS = [
+    {
+        "name": "research_company",
+        "title": "Research a Company",
+        "description": "Scrape and analyze a company website for competitive intelligence. Use the pry_scrape and pry_enrich tools to gather data, then summarize findings.",
+        "arguments": [
+            {
+                "name": "company_name",
+                "description": "Company name (e.g., 'Anthropic')",
+                "required": True,
+            },
+            {
+                "name": "website",
+                "description": "Company website URL (e.g., 'https://anthropic.com')",
+                "required": True,
+            },
+        ],
+    },
+    {
+        "name": "compare_products",
+        "title": "Compare Products",
+        "description": "Scrape product pages from multiple e-commerce sites and create a side-by-side comparison. Use pry_template tool with amazon-product, walmart-product, etc.",
+        "arguments": [
+            {
+                "name": "product_query",
+                "description": "What to search for (e.g., 'wireless headphones')",
+                "required": True,
+            },
+            {
+                "name": "sites",
+                "description": "Sites to compare (e.g., ['amazon', 'walmart', 'target'])",
+                "required": True,
+            },
+        ],
+    },
+    {
+        "name": "extract_contacts",
+        "title": "Extract Contacts from a Website",
+        "description": "Crawl a website and extract all email addresses, phone numbers, and social media handles using CSS selectors and regex patterns.",
+        "arguments": [
+            {"name": "url", "description": "Starting URL", "required": True},
+            {
+                "name": "max_pages",
+                "description": "Maximum pages to crawl (default: 20)",
+                "required": False,
+            },
+        ],
+    },
+    {
+        "name": "monitor_competitor",
+        "title": "Monitor a Competitor",
+        "description": "Set up continuous monitoring for a competitor's pricing, product listings, or content. Get notified via webhook on changes.",
+        "arguments": [
+            {"name": "url", "description": "Competitor URL to monitor", "required": True},
+            {
+                "name": "what_to_track",
+                "description": "What to track (e.g., 'product prices', 'blog posts', 'job listings')",
+                "required": True,
+            },
+            {"name": "webhook_url", "description": "Where to send notifications", "required": True},
+        ],
+    },
+    {
+        "name": "analyze_article",
+        "title": "Analyze an Article",
+        "description": "Scrape an article, summarize key points, and provide structured analysis (sentiment, entities, key claims).",
+        "arguments": [
+            {"name": "url", "description": "Article URL", "required": True},
+            {
+                "name": "focus",
+                "description": "What to focus on (e.g., 'financial implications', 'competitive threats')",
+                "required": False,
+            },
+        ],
+    },
+    {
+        "name": "extract_table",
+        "title": "Extract a Data Table",
+        "description": "Find and extract a specific data table from a webpage. Returns structured CSV/JSON rows.",
+        "arguments": [
+            {"name": "url", "description": "Page URL", "required": True},
+            {
+                "name": "table_description",
+                "description": "Description of the table you want (e.g., 'product pricing table', 'league standings')",
+                "required": True,
+            },
+        ],
+    },
+    {
+        "name": "scrape_with_schema",
+        "title": "Scrape with Custom Schema",
+        "description": "Scrape any URL with a custom JSON schema. Use pry_extract or pry_scrape with a schema definition.",
+        "arguments": [
+            {"name": "url", "description": "URL to scrape", "required": True},
+            {
+                "name": "fields",
+                "description": "Comma-separated list of fields to extract (e.g., 'title,price,description,image')",
+                "required": True,
+            },
+        ],
+    },
+]

pry_mcp/sdk_server.py

@@ -0,0 +1,89 @@
+"""Pry - MCP SDK server implementation.
+
+Uses the official MCP Python SDK when available.
+Contains create_server(), register_all(), and main().
+
+Split from mcp_production.py.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from pry_mcp.constants import SERVER_NAME, SERVER_VERSION
+from pry_mcp.fallback_server import make_fallback_server
+from pry_mcp.resources import PRY_PROMPTS, PRY_RESOURCES
+from pry_mcp.tools import PRY_TOOLS
+
+logger = logging.getLogger(__name__)
+
+# Try to import official MCP SDK
+try:
+    import mcp.server.session  # noqa: F401
+    from mcp.server import Server
+    from mcp.server.stdio import stdio_server
+    from mcp.types import (  # noqa: F401
+        Annotations,
+        BlobResourceContents,
+        CallToolRequest,
+        CallToolResult,
+        CompleteResult,
+        Completion,
+        CompletionArgument,
+        CompletionContext,
+        EmptyResult,
+        GetPromptResult,
+        ImageContent,
+        ListPromptsResult,
+        ListResourcesResult,
+        ListToolsResult,
+        LoggingLevel,
+        PaginatedRequest,
+        ReadResourceRequest,
+        ReadResourceResult,
+        Resource,
+        ServerResult,
+        TextContent,
+        TextResourceContents,
+        Tool,
+    )
+
+    _has_mcp_sdk = True
+except ImportError:
+    _has_mcp_sdk = False
+
+def create_server() -> Any:
+    """Create the MCP server (uses official SDK if available, fallback otherwise)."""
+    if _has_mcp_sdk:
+        # TODO: register tools/resources/prompts using official SDK decorators
+        # and return the SDK server when tool registration is complete.
+        Server(SERVER_NAME, SERVER_VERSION)
+    return make_fallback_server()
+
+
+def register_all(server: Any) -> None:
+    """Register all Pry tools, resources, and prompts."""
+    for tool in PRY_TOOLS:
+        server.register_tool(tool["name"], tool)
+    for resource in PRY_RESOURCES:
+        server.register_resource(resource["uri"], resource)
+    for prompt in PRY_PROMPTS:
+        server.register_prompt(prompt["name"], prompt)
+
+
+async def main() -> None:
+    """Run the MCP server."""
+    server = create_server()
+    register_all(server)
+    if _has_mcp_sdk:
+        async with stdio_server() as (read_stream, write_stream):
+            await server.run(read_stream, write_stream, server.create_initialization_options())
+    else:
+        await server.run_stdio()

pry_mcp/tools.py

@@ -0,0 +1,281 @@
+"""Pry - MCP tool definitions.
+
+PRY_TOOLS list with all scraper templates.
+
+Split from mcp_production.py.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+PRY_TOOLS = [
+    {
+        "name": "pry_scrape",
+        "description": "Scrape a URL to clean markdown. Bypasses Cloudflare, DataDome, Akamai, and other anti-bot systems automatically using a 9-tier fallback (direct → cloudscraper → FlareSolverr → Playwright → Googlebot → Archive.org → Google Cache → Tor → undetected-chromedriver).",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "url": {"type": "string", "format": "uri", "description": "URL to scrape"},
+                "bypass_cloudflare": {
+                    "type": "boolean",
+                    "default": True,
+                    "description": "Use FlareSolverr + Playwright fallback",
+                },
+                "js_render": {
+                    "type": "boolean",
+                    "default": False,
+                    "description": "Render JavaScript (slower but handles SPAs)",
+                },
+                "extract_schema": {
+                    "type": "object",
+                    "description": "Optional JSON schema for structured extraction (e.g., {fields: [{name, selector, type}]})",
+                },
+            },
+            "required": ["url"],
+        },
+        "outputSchema": {
+            "type": "object",
+            "properties": {
+                "success": {"type": "boolean"},
+                "data": {
+                    "type": "object",
+                    "properties": {
+                        "url": {"type": "string"},
+                        "title": {"type": "string"},
+                        "content": {"type": "string", "description": "Clean markdown content"},
+                        "links": {"type": "array", "items": {"type": "string"}},
+                        "images": {"type": "array", "items": {"type": "string"}},
+                        "status_code": {"type": "integer"},
+                    },
+                },
+            },
+            "required": ["success"],
+        },
+    },
+    {
+        "name": "pry_crawl",
+        "description": "Crawl a website starting from a URL. Discovers and scrapes linked pages up to max_pages with BFS/DFS traversal.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "url": {"type": "string", "format": "uri"},
+                "max_pages": {
+                    "type": "integer",
+                    "default": 10,
+                    "minimum": 1,
+                    "maximum": 10000,
+                },
+                "max_depth": {
+                    "type": "integer",
+                    "default": 2,
+                    "minimum": 0,
+                    "maximum": 20,
+                },
+                "include_paths": {
+                    "type": "array",
+                    "items": {"type": "string"},
+                    "description": "URL path patterns to include (e.g., ['/docs/', '/blog/'])",
+                },
+                "exclude_paths": {"type": "array", "items": {"type": "string"}},
+            },
+            "required": ["url"],
+        },
+        "outputSchema": {
+            "type": "object",
+            "properties": {
+                "success": {"type": "boolean"},
+                "data": {
+                    "type": "object",
+                    "properties": {
+                        "pages": {"type": "array", "items": {"type": "object"}},
+                        "total_pages": {"type": "integer"},
+                    },
+                },
+            },
+        },
+    },
+    {
+        "name": "pry_extract",
+        "description": "Extract structured data from a URL using CSS selectors. 10x cheaper than LLM extraction. Returns JSON matching the schema.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "url": {"type": "string", "format": "uri"},
+                "schema": {
+                    "type": "object",
+                    "description": "Extraction schema with base_selector and fields array",
+                    "properties": {
+                        "name": {"type": "string"},
+                        "base_selector": {"type": "string"},
+                        "fields": {
+                            "type": "array",
+                            "items": {
+                                "type": "object",
+                                "properties": {
+                                    "name": {"type": "string"},
+                                    "selector": {"type": "string"},
+                                    "type": {
+                                        "type": "string",
+                                        "enum": [
+                                            "text",
+                                            "attribute",
+                                            "html",
+                                            "nested",
+                                            "count",
+                                            "exists",
+                                            "regex",
+                                        ],
+                                    },
+                                    "attribute": {"type": "string"},
+                                    "transform": {"type": "string"},
+                                },
+                                "required": ["name", "selector"],
+                            },
+                        },
+                    },
+                    "required": ["name", "fields"],
+                },
+            },
+            "required": ["url", "schema"],
+        },
+        "outputSchema": {
+            "type": "object",
+            "properties": {
+                "success": {"type": "boolean"},
+                "data": {
+                    "type": "object",
+                    "properties": {
+                        "items": {"type": "array", "items": {"type": "object"}},
+                    },
+                },
+            },
+            "required": ["success"],
+        },
+    },
+    {
+        "name": "pry_template",
+        "description": "Execute one of 110+ pre-built scraper templates (amazon-product, walmart-product, linkedin-profile, etc.). Returns structured data matching the site's schema.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "template_id": {
+                    "type": "string",
+                    "description": "Template ID (e.g., 'amazon-product', 'walmart-product', 'linkedin-profile')",
+                },
+                "url": {"type": "string", "format": "uri"},
+            },
+            "required": ["template_id", "url"],
+        },
+    },
+    {
+        "name": "pry_search_templates",
+        "description": "Search across all 110+ available scraper templates by keyword, category, or site.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "q": {
+                    "type": "string",
+                    "description": "Search query (e.g., 'amazon', 'jobs', 'real estate')",
+                },
+                "category": {"type": "string", "description": "Filter by category"},
+            },
+            "required": ["q"],
+        },
+    },
+    {
+        "name": "pry_monitor",
+        "description": "Create a content change monitor. Get notified via webhook, Slack, or Discord when a page changes. AI judges if changes are meaningful.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string"},
+                "url": {"type": "string", "format": "uri"},
+                "schedule_cron": {
+                    "type": "string",
+                    "default": "0 */6 * * *",
+                    "description": "Cron expression (default: every 6 hours)",
+                },
+                "webhook_url": {"type": "string", "format": "uri"},
+                "goal": {
+                    "type": "string",
+                    "description": "Natural language goal for AI change judging (e.g., 'price changes matter, not descriptions')",
+                },
+            },
+            "required": ["name", "url"],
+        },
+    },
+    {
+        "name": "pry_compliance",
+        "description": "Check legal compliance for scraping a URL. Analyzes robots.txt, ToS, GDPR/CCPA, and PII exposure. Returns green/yellow/red risk score with recommendations.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {"url": {"type": "string", "format": "uri"}},
+            "required": ["url"],
+        },
+    },
+    {
+        "name": "pry_enrich",
+        "description": "Enrich a URL with company info, tech stack detection, social profiles, and competitive intelligence.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {"url": {"type": "string", "format": "uri"}},
+            "required": ["url"],
+        },
+    },
+    {
+        "name": "pry_parse_document",
+        "description": "Parse a document (PDF, DOCX, image) and extract text/tables. Uses pdfplumber, Tesseract OCR, and python-docx.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {"url": {"type": "string", "format": "uri"}},
+            "required": ["url"],
+        },
+    },
+    {
+        "name": "pry_screenshot",
+        "description": "Take a full-page screenshot of a URL. Returns base64-encoded PNG. Useful for visual analysis and OCR.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "url": {"type": "string", "format": "uri"},
+                "full_page": {"type": "boolean", "default": True},
+            },
+            "required": ["url"],
+        },
+    },
+    {
+        "name": "pry_x402_pricing",
+        "description": "Get current x402 pay-per-call pricing for all Pry operations. Shows the cost in USDC atomic units for each tool.",
+        "inputSchema": {"type": "object", "properties": {}, "required": []},
+        "outputSchema": {
+            "type": "object",
+            "properties": {
+                "success": {"type": "boolean"},
+                "data": {
+                    "type": "object",
+                    "properties": {
+                        "pricing": {"type": "object"},
+                        "wallet": {"type": "string"},
+                        "facilitator": {"type": "string"},
+                        "supported_networks": {"type": "array", "items": {"type": "string"}},
+                    },
+                },
+            },
+            "required": ["success"],
+        },
+    },
+    {
+        "name": "pry_referrals",
+        "description": "Get Pry's referral catalog (60+ providers across LLM, hosting, email, monitoring, proxies, devtools). Each link includes our affiliate ID for revenue attribution.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {"category": {"type": "string"}},
+            "required": [],
+        },
+    },
+]

pryfile.py

@@ -25,6 +25,7 @@ jobs:
 
 import json
 import os
+from typing import Any
 
 import yaml
 
@@ -34,7 +35,7 @@ class Pryfile:
 
     def __init__(self, path: str = "pry.yml"):
         self.path = path
-        self.jobs = []
+        self.jobs: list[Any] = []
         self._load()
 
     def _load(self):

pyproject.toml

@@ -102,7 +102,6 @@ warn_unused_ignores = false
 # TODO: Pry was built without strict typing; re-enable incrementally.
 # Tracking issue: re-enable mypy strict mode after router refactor.
 disable_error_code = [
-    "var-annotated",
     "dict-item",
     "index",
     "attr-defined",

resilience.py

@@ -0,0 +1,228 @@
+"""Resilience primitives for Pry scrapers.
+
+Provides circuit breakers and per-domain tier memory so the fallback chain
+can skip unhealthy services and prefer the cheapest successful strategy for
+a given domain.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+from __future__ import annotations
+
+import logging
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Any, ClassVar
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_FAILURE_THRESHOLD = 5
+DEFAULT_RECOVERY_SECONDS = 30
+DEFAULT_HALF_OPEN_MAX_TRIES = 2
+DOMAIN_MEMORY_MAX_AGE_SECONDS = 3600
+
+
+class CircuitState:
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+
+@dataclass
+class CircuitBreaker:
+    """Simple circuit breaker with exponential cooldown."""
+
+    name: str
+    failure_threshold: int = DEFAULT_FAILURE_THRESHOLD
+    recovery_seconds: float = DEFAULT_RECOVERY_SECONDS
+    half_open_max_tries: int = DEFAULT_HALF_OPEN_MAX_TRIES
+    state: str = CircuitState.CLOSED
+    failures: int = 0
+    successes: int = 0
+    last_failure_at: float = 0.0
+    half_open_attempts: int = 0
+
+    def is_allowed(self) -> bool:
+        now = time.time()
+        if self.state == CircuitState.CLOSED:
+            return True
+        if self.state == CircuitState.OPEN:
+            if now - self.last_failure_at >= self.recovery_seconds:
+                self.state = CircuitState.HALF_OPEN
+                self.half_open_attempts = 0
+                logger.info("circuit_breaker_half_open", extra={"service": self.name})
+                return True
+            return False
+        # HALF_OPEN
+        return self.half_open_attempts < self.half_open_max_tries
+
+    def record_success(self) -> None:
+        if self.state == CircuitState.HALF_OPEN:
+            self.half_open_attempts += 1
+            if self.half_open_attempts >= self.half_open_max_tries:
+                self._close()
+            return
+        self.successes += 1
+        # Decay failures slowly on success to forgive transient blips.
+        if self.failures > 0:
+            self.failures -= 1
+
+    def record_failure(self) -> None:
+        self.failures += 1
+        self.last_failure_at = time.time()
+        if self.state == CircuitState.HALF_OPEN:
+            self._open()
+            return
+        if self.failures >= self.failure_threshold:
+            self._open()
+
+    def _open(self) -> None:
+        if self.state != CircuitState.OPEN:
+            logger.warning(
+                "circuit_breaker_open",
+                extra={"service": self.name, "failures": self.failures},
+            )
+        self.state = CircuitState.OPEN
+        self.half_open_attempts = 0
+
+    def _close(self) -> None:
+        logger.info("circuit_breaker_closed", extra={"service": self.name})
+        self.state = CircuitState.CLOSED
+        self.failures = 0
+        self.half_open_attempts = 0
+
+
+@dataclass
+class TierRecord:
+    successes: int = 0
+    failures: int = 0
+    last_success_at: float = 0.0
+    last_failure_at: float = 0.0
+    total_latency: float = 0.0
+
+    @property
+    def score(self) -> float:
+        """Higher score = more preferred. Balances success rate and recency."""
+        total = self.successes + self.failures
+        if total == 0:
+            return 0.0
+        success_rate = self.successes / total
+        # Recency bonus: 1.0 if succeeded in last 5 minutes, decaying.
+        recency = 0.0
+        if self.last_success_at > 0:
+            age = time.time() - self.last_success_at
+            recency = max(0.0, 1.0 - age / DOMAIN_MEMORY_MAX_AGE_SECONDS)
+        return success_rate + recency * 0.5
+
+
+@dataclass
+class DomainTierMemory:
+    """Remember which tiers work best per domain."""
+
+    data: dict[str, dict[str, TierRecord]] = field(default_factory=lambda: defaultdict(dict))
+
+    def record(self, domain: str, tier: str, success: bool, latency: float = 0.0) -> None:
+        rec = self.data[domain].setdefault(tier, TierRecord())
+        if success:
+            rec.successes += 1
+            rec.last_success_at = time.time()
+        else:
+            rec.failures += 1
+            rec.last_failure_at = time.time()
+        rec.total_latency += latency
+
+    def best_tier(self, domain: str, candidates: list[str]) -> str | None:
+        records = self.data.get(domain, {})
+        if not records:
+            return None
+        # Only consider tiers with at least one success and no recent failures.
+        now = time.time()
+        eligible = [
+            (tier, rec)
+            for tier, rec in records.items()
+            if rec.successes > 0
+            and (rec.last_failure_at == 0 or now - rec.last_failure_at > 300)
+            and tier in candidates
+        ]
+        if not eligible:
+            return None
+        eligible.sort(key=lambda item: item[1].score, reverse=True)
+        return eligible[0][0]
+
+    def to_dict(self) -> dict[str, dict[str, dict[str, Any]]]:
+        return {
+            domain: {
+                tier: {
+                    "successes": rec.successes,
+                    "failures": rec.failures,
+                    "last_success_at": rec.last_success_at,
+                    "last_failure_at": rec.last_failure_at,
+                    "score": round(rec.score, 3),
+                }
+                for tier, rec in tiers.items()
+            }
+            for domain, tiers in self.data.items()
+        }
+
+
+class ResilienceRegistry:
+    """Global registry for circuit breakers and domain tier memory."""
+
+    # External services that should have circuit breakers.
+    SERVICE_NAMES: ClassVar[list[str]] = [
+        "flaresolverr",
+        "ollama",
+        "openrouter",
+        "redis",
+    ]
+
+    def __init__(self) -> None:
+        self.breakers: dict[str, CircuitBreaker] = {
+            name: CircuitBreaker(name=name) for name in self.SERVICE_NAMES
+        }
+        self.domain_memory = DomainTierMemory()
+
+    def breaker(self, name: str) -> CircuitBreaker:
+        if name not in self.breakers:
+            self.breakers[name] = CircuitBreaker(name=name)
+        return self.breakers[name]
+
+    def is_service_allowed(self, name: str) -> bool:
+        return self.breaker(name).is_allowed()
+
+    def record_service_result(self, name: str, success: bool) -> None:
+        breaker = self.breaker(name)
+        if success:
+            breaker.record_success()
+        else:
+            breaker.record_failure()
+
+    def record_domain_tier_result(
+        self, domain: str, tier: str, success: bool, latency: float = 0.0
+    ) -> None:
+        self.domain_memory.record(domain, tier, success, latency)
+
+    def best_tier_for_domain(self, domain: str, candidates: list[str]) -> str | None:
+        return self.domain_memory.best_tier(domain, candidates)
+
+    def stats(self) -> dict[str, Any]:
+        return {
+            "circuit_breakers": {
+                name: {
+                    "state": cb.state,
+                    "failures": cb.failures,
+                    "successes": cb.successes,
+                    "last_failure_at": cb.last_failure_at,
+                }
+                for name, cb in self.breakers.items()
+            },
+            "domain_memory": self.domain_memory.to_dict(),
+        }
+
+
+# Global singleton used by scrapers.
+registry = ResilienceRegistry()

retry.py

@@ -0,0 +1,246 @@
+"""Retry/backoff helpers for Pry scrapers.
+
+Provides configurable async retry with exponential backoff, jitter, and
+circuit-breaker integration so external service calls degrade gracefully.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+from __future__ import annotations
+
+import asyncio
+import enum
+import logging
+import random
+import time
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any, TypeVar
+
+import resilience
+
+logger = logging.getLogger(__name__)
+
+# ── Types ─────────────────────────────────────────────────────────
+F = TypeVar("F", bound=Callable[..., Awaitable[Any]])
+
+
+class BackoffStrategy(enum.StrEnum):
+    """Available backoff strategies for retry delays."""
+
+    EXPONENTIAL = "exponential"
+    LINEAR = "linear"
+    FIXED = "fixed"
+
+
+class Jitter(enum.StrEnum):
+    """Jitter modes for retry delays.
+
+    - NONE: no jitter, pure backoff.
+    - FULL: random uniform between 0 and the computed delay.
+    - EQUAL: the delay is halved and jitter is applied to each half.
+    - DECORRELATED: previous_delay * random(1, 3), capped at max_delay.
+    """
+
+    NONE = "none"
+    FULL = "full"
+    EQUAL = "equal"
+    DECORRELATED = "decorrelated"
+
+
+# ── Config ────────────────────────────────────────────────────────
+@dataclass
+class RetryConfig:
+    """Configuration for async retry behavior.
+
+    Attributes:
+        max_retries: Maximum number of retry attempts (does not include the
+            initial call). Default 3 means up to 1 initial + 3 retries = 4
+            total attempts.
+        max_time: Maximum wall-clock seconds to keep retrying. 0 = no limit.
+        base_delay: Base delay in seconds for backoff calculations.
+        max_delay: Maximum delay in seconds between retries.
+        strategy: Backoff strategy (exponential, linear, fixed).
+        jitter: Jitter mode (none, full, equal, decorrelated).
+        retryable_exceptions: Tuple of exception types that trigger retry.
+            If None, all exceptions are retried.
+        circuit_breaker: Name of a ResilienceRegistry circuit breaker to
+            consult before retrying. If the breaker is open, retry is skipped
+            and the last error is re-raised. None = no circuit breaker.
+    """
+
+    max_retries: int = 3
+    max_time: float = 0.0
+    base_delay: float = 1.0
+    max_delay: float = 60.0
+    strategy: BackoffStrategy = BackoffStrategy.EXPONENTIAL
+    jitter: Jitter = Jitter.FULL
+    retryable_exceptions: tuple[type[Exception], ...] | None = None
+    circuit_breaker: str | None = None
+
+
+# ── Wait calculators ──────────────────────────────────────────────
+def _calculate_delay(
+    attempt: int,
+    config: RetryConfig,
+    *,
+    previous_delay: float = 0.0,
+) -> float:
+    """Compute the base delay for the given attempt number."""
+    if config.strategy == BackoffStrategy.FIXED:
+        delay = config.base_delay
+    elif config.strategy == BackoffStrategy.LINEAR:
+        delay = config.base_delay * (attempt + 1)
+    else:  # EXPONENTIAL
+        delay = config.base_delay * (2**attempt)
+
+    delay = min(delay, config.max_delay)
+    return _apply_jitter(delay, config.jitter, previous_delay)
+
+
+def _apply_jitter(delay: float, mode: Jitter, previous_delay: float = 0.0) -> float:
+    """Apply jitter to a delay value."""
+    if mode == Jitter.NONE:
+        return delay
+    if mode == Jitter.FULL:
+        return random.uniform(0, delay)
+    if mode == Jitter.EQUAL:
+        half = delay / 2.0
+        return half + random.uniform(0, half)
+    if mode == Jitter.DECORRELATED:
+        if previous_delay <= 0:
+            return delay
+        return min(previous_delay * random.uniform(1, 3), 60.0)
+    return delay
+
+
+# ── Public retry helpers ──────────────────────────────────────────
+async def async_retry(
+    fn: Callable[..., Awaitable[Any]],
+    *args: Any,
+    config: RetryConfig | None = None,
+    **kwargs: Any,
+) -> Any:
+    """Call an async function with configurable retry and backoff.
+
+    Args:
+        fn: Async function to call.
+        *args: Positional arguments passed to fn.
+        config: Retry configuration. If None, uses defaults (3 retries,
+            exponential backoff, full jitter).
+        **kwargs: Keyword arguments passed to fn.
+
+    Returns:
+        The result of fn(*args, **kwargs) on success.
+
+    Raises:
+        The last exception raised by fn if all retries are exhausted, or
+        if a non-retryable exception (not in config.retryable_exceptions)
+        is raised.
+    """
+    cfg = config or RetryConfig()
+    last_exc: Exception | None = None
+    start_time = time.monotonic()
+    previous_delay = 0.0
+
+    # The first call is the initial attempt, not a retry.
+    total_attempts = cfg.max_retries + 1
+
+    for attempt in range(total_attempts):
+        # If a circuit breaker is configured and open, skip immediately.
+        if cfg.circuit_breaker and not resilience.registry.is_service_allowed(cfg.circuit_breaker):
+            logger.debug(
+                "retry_circuit_open",
+                extra={
+                    "service": cfg.circuit_breaker,
+                    "fn": fn.__name__,
+                    "attempt": attempt,
+                },
+            )
+            if last_exc is not None:
+                raise last_exc
+            raise RuntimeError(
+                f"Circuit breaker '{cfg.circuit_breaker}' is open \u2014 retry skipped"
+            )
+
+        try:
+            result = await fn(*args, **kwargs)
+            # Record success to the circuit breaker.
+            if cfg.circuit_breaker:
+                resilience.registry.record_service_result(cfg.circuit_breaker, True)
+            return result
+        except Exception as e:
+            last_exc = e
+
+            # Record failure to the circuit breaker.
+            if cfg.circuit_breaker:
+                resilience.registry.record_service_result(cfg.circuit_breaker, False)
+
+            # Check if this exception type is retryable.
+            if cfg.retryable_exceptions is not None and not isinstance(e, cfg.retryable_exceptions):
+                raise
+
+            # If this was the last attempt, break out to re-raise.
+            if attempt >= cfg.max_retries:
+                break
+
+            # Check max wall-clock time.
+            if cfg.max_time > 0 and (time.monotonic() - start_time) >= cfg.max_time:
+                logger.warning(
+                    "retry_max_time_exceeded",
+                    extra={
+                        "fn": fn.__name__,
+                        "max_time": cfg.max_time,
+                        "attempts": attempt + 1,
+                    },
+                )
+                break
+
+            # Wait before next attempt.
+            delay = _calculate_delay(attempt, cfg, previous_delay=previous_delay)
+            previous_delay = delay
+            logger.info(
+                "retry_waiting",
+                extra={
+                    "fn": fn.__name__,
+                    "attempt": attempt + 1,
+                    "max_retries": cfg.max_retries,
+                    "delay_seconds": round(delay, 2),
+                    "error": str(e)[:80],
+                },
+            )
+            await asyncio.sleep(delay)
+
+    # All retries exhausted.
+    if last_exc is not None:
+        raise last_exc
+
+    raise RuntimeError("retry_exhausted_no_exception")
+
+
+def make_retry_decorator(
+    config: RetryConfig | None = None,
+) -> Callable[[F], F]:
+    """Create a decorator that wraps an async function with retry logic.
+
+    Example:
+        @make_retry_decorator(RetryConfig(max_retries=5, base_delay=0.5))
+        async def fetch_data(url: str) -> dict:
+            ...
+    """
+    cfg = config or RetryConfig()
+
+    def decorator(fn: F) -> F:
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            return await async_retry(fn, *args, config=cfg, **kwargs)
+
+        wrapper.__name__ = fn.__name__
+        wrapper.__qualname__ = getattr(fn, "__qualname__", fn.__name__)
+        wrapper.__doc__ = fn.__doc__
+        wrapper.__module__ = getattr(fn, "__module__", "")
+        return wrapper  # type: ignore[return-value]
+
+    return decorator

routers/config.py

@@ -1,6 +1,7 @@
-"""Pry — Config router (remaining api.py routes).
+"""Pry — Config router (runtime configuration management).
 
-Auto-extracted from api.py during the router-split refactor.
+Provides endpoints to read and update the unified Pry settings object.
+Runtime changes are persisted to the configured JSON file.
 """
 
 # SPDX-License-Identifier: MIT
@@ -13,37 +14,33 @@ from typing import Any
 
 from fastapi import APIRouter, Body
 
-from mconfig import PryConfig
+from settings import settings
 
 logger = logging.getLogger(__name__)
 
 router = APIRouter(tags=["Config"])
 
+
 @router.get("/v1/config", tags=["Config"], summary="Get current Pry configuration")
 async def get_config() -> dict[str, Any]:
     """Get current Pry configuration."""
-    return {"success": True, "data": config.to_dict()}
+    return {"success": True, "data": settings.to_api_dict()}
 
 
 @router.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime")
 async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]:
     """Update Pry configuration at runtime."""
-    result = config.update(updates)
+    result = settings.update(updates)
     return {"success": True, "data": result}
 
 
 @router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests")
 async def enable_tor() -> dict[str, Any]:
     """Enable Tor routing for all requests."""
-    result = config.update(
-        {"tor": {"enabled": True}, "proxy": {"enabled": True, "url": "socks5://tor:9050"}}
-    )
+    result = settings.enable_tor()
     resp_data = {
         "success": True,
         "data": result,
         "note": "Run 'docker compose --profile tor up -d' to start Tor container",
     }
     return resp_data
-
-
-config = PryConfig()

routers/vision.py

@@ -19,6 +19,7 @@ from fastapi import APIRouter, Body
 from client import get_client
 from deps import automator
 from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError
+from resilience import registry
 from settings import settings
 
 logger = logging.getLogger(__name__)
@@ -171,6 +172,8 @@ async def _call_vision_api(
     key = _get_or_key()
     if not key:
         return None, None, "OPENROUTER_API_KEY not set"
+    if not registry.is_service_allowed("openrouter"):
+        return None, None, "OpenRouter circuit breaker open"
 
     # Accept either data URI or raw base64
     data_uri = image_b64 if image_b64.startswith("data:") else f"data:image/png;base64,{image_b64}"
@@ -203,6 +206,7 @@ async def _call_vision_api(
     )
     resp.raise_for_status()
     body = resp.json()
+    registry.record_service_result("openrouter", True)
     return body["choices"][0]["message"]["content"], model, None
 
 

scraper.py

@@ -19,10 +19,10 @@ import httpx
 import trafilatura
 from trafilatura.settings import use_config
 
-from mconfig import PryConfig
-
-FLARESOLVERR_URL = "http://flaresolverr:8191/v1"
-config = PryConfig()
+from errors import InvalidRequestError
+from settings import settings
+from stealth_engine import StealthEngine
+from url_guard import validate_url
 
 USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
@@ -283,6 +283,10 @@ class PryScraper:
     async def scrape(self, url: str, options: dict | None = None) -> dict[str, Any]:
         if not url or not url.startswith(("http://", "https://")):
             return {"status": "error", "url": url, "error": "Invalid URL", "content": ""}
+        try:
+            validate_url(url)
+        except InvalidRequestError as e:
+            return {"status": "error", "url": url, "error": str(e), "content": ""}
 
         opts = options or {}
 
@@ -415,7 +419,7 @@ class PryScraper:
         return result
 
     async def _fetch_direct(self, url: str, headers: dict, timeout: int) -> str:
-        proxy_url = config.get_proxy_url()
+        proxy_url = settings.resolved_proxy_url
         async with httpx.AsyncClient(
             timeout=httpx.Timeout(timeout),
             follow_redirects=True,
@@ -424,7 +428,7 @@ class PryScraper:
             cookies=httpx.Cookies(),
             proxy=proxy_url,
         ) as client:
-            delay = random.uniform(*config.get("rotation.delay_range", [0.5, 3.0]))
+            delay = random.uniform(*settings.rotation_delay_range)
             await asyncio.sleep(delay)
             resp = await client.get(url)
             resp.raise_for_status()
@@ -433,7 +437,7 @@ class PryScraper:
     async def _fetch_via_flaresolverr(self, url: str, timeout: int) -> str:
         payload = {"cmd": "request.get", "url": url, "maxTimeout": timeout * 1000}
         async with httpx.AsyncClient(timeout=httpx.Timeout(timeout + 10)) as client:
-            resp = await client.post(FLARESOLVERR_URL, json=payload)
+            resp = await client.post(settings.flaresolverr_url, json=payload)
             resp.raise_for_status()
             data = resp.json()
             solution = data.get("solution", {})
@@ -470,17 +474,8 @@ class PryScraper:
                 geolocation={"latitude": 40.7128, "longitude": -74.0060},
             )
             # Stealth: override automation indicators
-            await context.add_init_script("""
-                Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
-                Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
-                Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
-                window.chrome = { runtime: {} };
-                // Override permissions query to avoid automation detection
-                const originalQuery = window.navigator.permissions.query;
-                window.navigator.permissions.query = (p) => (
-                    p.name === 'notifications' ? Promise.resolve({state: 'denied'}) : originalQuery(p)
-                );
-            """)
+            stealth_script = StealthEngine().generate_all()
+            await context.add_init_script(stealth_script)
             page = await context.new_page()
             # Human-like navigation: random wait before page load
             await asyncio.sleep(random.uniform(0.5, 2.0))
@@ -511,7 +506,9 @@ class PryScraper:
         opts = options or {}
         max_pages = opts.get("max_pages", 10)
         max_depth = opts.get("max_depth", 2)
-        visited, to_visit, results = set(), [(url, 0)], []
+        visited: set[str] = set()
+        to_visit: list[tuple[str, int]] = [(url, 0)]
+        results: list[Any] = []
 
         while to_visit and len(visited) < max_pages:
             current_url, depth = to_visit.pop(0)

settings.py

@@ -1,14 +1,60 @@
 """Pry — centralized settings via Pydantic BaseSettings.
-Every env var lives here. Import `settings` singleton everywhere else."""
+
+Every environment variable lives here. Import the `settings` singleton
+everywhere else.
+
+Legacy env vars without the `PRY_` prefix (e.g. `PROXY_URL`, `TOR_ENABLED`)
+are still accepted during a deprecation window.
+
+The settings object also loads and persists runtime overrides from a JSON
+config file (default `/app/config.json`).
+"""
 
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2026 Rug Munch Media LLC
 #
 # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
 # Licensed under MIT. See LICENSE.
+from __future__ import annotations
 
+import json
+from pathlib import Path
+from typing import Any, Literal
+
+from pydantic import AliasChoices, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+DEFAULT_CONFIG_FILE = Path("/app/config.json")
+
+# Map legacy nested config keys (from the old mconfig JSON file and API)
+# to flat attribute names on PrySettings.
+_NESTED_KEY_MAP: dict[str, str] = {
+    "proxy.enabled": "proxy_enabled",
+    "proxy.type": "proxy_type",
+    "proxy.url": "proxy_url",
+    "proxy.username": "proxy_username",
+    "proxy.password": "proxy_password",
+    "tor.enabled": "tor_enabled",
+    "tor.socks5_host": "tor_socks5_host",
+    "tor.socks5_port": "tor_socks5_port",
+    "tor.control_port": "tor_control_port",
+    "rotation.user_agent": "rotation_user_agent",
+    "rotation.ip": "ip_rotation",
+    "rotation.timing": "rotation_timing",
+    "rotation.delay_range": "rotation_delay_range",
+    "stealth.webdriver_override": "webdriver_override",
+    "stealth.canvas_noise": "canvas_noise",
+    "stealth.webrtc_disable": "webrtc_disable",
+    "stealth.geolocation_spoof": "geolocation_spoof",
+    "retry.max_attempts": "max_retries",
+    "retry.min_quality": "min_quality",
+    "retry.backoff": "retry_backoff",
+    "output.default_format": "default_format",
+    "output.max_chars": "max_chars",
+    "output.include_links": "include_links",
+    "rate_limit.rpm": "rate_limit_rpm",
+}
+
 
 class PrySettings(BaseSettings):
     model_config = SettingsConfigDict(
@@ -17,45 +63,261 @@ class PrySettings(BaseSettings):
         env_file_encoding="utf-8",
         case_sensitive=False,
         extra="ignore",
+        populate_by_name=True,
     )
 
     # Core
-    url: str = "http://localhost:8002"
+    app_name: str = "Pry"
+    app_version: str = "3.0.0"
+    environment: Literal["dev", "staging", "prod"] = "dev"
+    log_level: str = "INFO"
     host: str = "0.0.0.0"  # nosec B104
     port: int = 8002
-
-    # Ollama LLM endpoint
-    ollama_url: str = "http://100.104.130.92:11434"
-
-    # FlareSolverr Cloudflare bypass endpoint
-    flaresolverr_url: str = "http://flaresolverr:8191/v1"
-
-    # Webhook secret for job callbacks
-    webhook_secret: str = "pry-webhook-secret"
-
-    # API key for endpoint authentication (empty = disabled)
+    url: str = "http://localhost:8002"
     api_key: str = ""
 
-    # OpenRouter API key (optional, for vision models)
-    openrouter_api_key: str = ""
-
-    # Proxy / Tor
-    proxy_url: str = ""
-    proxy_type: str = "http"
-    proxy_username: str = ""
-    proxy_password: str = ""
-    tor_enabled: bool = False
-    tor_socks5_host: str = "tor"
-    tor_socks5_port: int = 9050
-
-    # Rotation
-    ip_rotation: str = ""
-    max_retries: int = 0
-    min_quality: int = 0
-    rate_limit_rpm: int = 0
-
-    # Redis (optional, for job queue persistence)
+    # Database / cache
+    database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry"
     redis_url: str = "redis://localhost:6379/0"
 
+    # LLM providers
+    ollama_url: str = "http://100.104.130.92:11434"
+    openrouter_api_key: str = ""
+    openai_api_key: str = ""
+    anthropic_api_key: str = ""
+    cohere_api_key: str = ""
 
+    # Services
+    flaresolverr_url: str = "http://flaresolverr:8191/v1"
+    webhook_secret: str = "pry-webhook-secret"
+
+    # Proxy (legacy env aliases for migration)
+    proxy_url: str = Field(
+        default="",
+        validation_alias=AliasChoices("PRY_PROXY_URL", "PROXY_URL"),
+    )
+    proxy_type: str = Field(
+        default="http",
+        validation_alias=AliasChoices("PRY_PROXY_TYPE", "PROXY_TYPE"),
+    )
+    proxy_username: str = Field(
+        default="",
+        validation_alias=AliasChoices("PRY_PROXY_USERNAME", "PROXY_USERNAME"),
+    )
+    proxy_password: str = Field(
+        default="",
+        validation_alias=AliasChoices("PRY_PROXY_PASSWORD", "PROXY_PASSWORD"),
+    )
+    proxy_enabled: bool = Field(
+        default=False,
+        validation_alias=AliasChoices("PRY_PROXY_ENABLED", "PROXY_ENABLED"),
+    )
+
+    # Tor (legacy env aliases)
+    tor_enabled: bool = Field(
+        default=False,
+        validation_alias=AliasChoices("PRY_TOR_ENABLED", "TOR_ENABLED"),
+    )
+    tor_socks5_host: str = Field(
+        default="tor",
+        validation_alias=AliasChoices("PRY_TOR_SOCKS5_HOST", "TOR_SOCKS5_HOST"),
+    )
+    tor_socks5_port: int = Field(
+        default=9050,
+        validation_alias=AliasChoices("PRY_TOR_SOCKS5_PORT", "TOR_SOCKS5_PORT"),
+    )
+    tor_control_port: int = 9051
+
+    # Rotation / retry (legacy env aliases)
+    rotation_user_agent: str = "rotate"
+    ip_rotation: str = Field(
+        default="off",
+        validation_alias=AliasChoices("PRY_IP_ROTATION", "IP_ROTATION"),
+    )
+    rotation_timing: str = "random"
+    rotation_delay_range: tuple[float, float] = (0.5, 3.0)
+    max_retries: int = Field(
+        default=3,
+        validation_alias=AliasChoices("PRY_MAX_RETRIES", "MAX_RETRIES"),
+    )
+    min_quality: int = Field(
+        default=20,
+        validation_alias=AliasChoices("PRY_MIN_QUALITY", "MIN_QUALITY"),
+    )
+    retry_backoff: str = "exponential"
+    rate_limit_rpm: int = Field(
+        default=120,
+        validation_alias=AliasChoices("PRY_RATE_LIMIT_RPM", "RATE_LIMIT_RPM"),
+    )
+
+    # Stealth
+    stealth_enabled: bool = True
+    random_user_agent: bool = True
+    webdriver_override: bool = True
+    canvas_noise: bool = True
+    webrtc_disable: bool = True
+    geolocation_spoof: bool = True
+    min_delay_ms: int = 500
+    max_delay_ms: int = 3000
+
+    # Output
+    default_format: str = "markdown"
+    max_chars: int = 100000
+    include_links: bool = True
+
+    # Storage
+    screenshot_dir: Path = Path("/tmp/pry-screenshots")  # nosec B108
+    cache_ttl_seconds: int = 3600
+    config_file: Path = Field(
+        default=DEFAULT_CONFIG_FILE,
+        validation_alias=AliasChoices("PRY_CONFIG_FILE", "CONFIG_FILE"),
+    )
+
+    # x402 / MCP
+    x402_enabled: bool = False
+    x402_pay_to: str = ""
+    mcp_enabled: bool = True
+    mcp_tools_count: int = 8
+
+    # Rate limits (legacy Settings class)
+    rate_limit_per_domain: int = 10
+    rate_limit_per_ip: int = 60
+
+    def model_post_init(self, __context: Any) -> None:
+        """Load runtime overrides from the JSON config file after env parsing."""
+        self._load_config_file()
+
+    def _load_config_file(self) -> None:
+        path = self.config_file
+        if not path.exists():
+            return
+        try:
+            with open(path, encoding="utf-8") as f:
+                data = json.load(f)
+        except (OSError, json.JSONDecodeError):
+            return
+        if not isinstance(data, dict):
+            return
+        self._apply_nested_updates(data)
+
+    def _apply_nested_updates(self, data: dict[str, Any]) -> None:
+        """Flatten nested config updates and set attributes on self."""
+        flat: dict[str, Any] = {}
+
+        def _flatten(obj: Any, prefix: str = "") -> None:
+            if isinstance(obj, dict):
+                for key, value in obj.items():
+                    new_prefix = f"{prefix}.{key}" if prefix else key
+                    _flatten(value, new_prefix)
+            else:
+                flat[prefix] = obj
+
+        _flatten(data)
+        for nested_key, value in flat.items():
+            attr = _NESTED_KEY_MAP.get(nested_key, nested_key)
+            if hasattr(self, attr):
+                if value is None:
+                    continue
+                current = getattr(self, attr)
+                expected_type = type(current)
+                try:
+                    if expected_type is tuple and isinstance(value, list):
+                        setattr(self, attr, tuple(value))
+                    else:
+                        setattr(self, attr, expected_type(value))
+                except (TypeError, ValueError):
+                    setattr(self, attr, value)
+
+    @property
+    def resolved_proxy_chain(self) -> list[str]:
+        """Return proxy chain: Tor -> user proxy -> direct."""
+        proxies: list[str] = []
+        if self.tor_enabled:
+            proxies.append(f"socks5://{self.tor_socks5_host}:{self.tor_socks5_port}")
+        if self.proxy_enabled and self.proxy_url:
+            if self.proxy_username:
+                netloc = self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url
+                proxies.append(
+                    f"{self.proxy_type}://{self.proxy_username}:{self.proxy_password}@{netloc}"
+                )
+            else:
+                proxies.append(self.proxy_url)
+        return proxies
+
+    @property
+    def resolved_proxy_url(self) -> str | None:
+        """Return the first proxy in the chain, or None for direct connection."""
+        chain = self.resolved_proxy_chain
+        return chain[0] if chain else None
+
+    def to_api_dict(self) -> dict[str, Any]:
+        """Return the nested config representation used by the /v1/config API."""
+        return {
+            "proxy": {
+                "enabled": self.proxy_enabled,
+                "type": self.proxy_type,
+                "url": self.proxy_url,
+                "username": self.proxy_username,
+                "password": self.proxy_password,
+            },
+            "tor": {
+                "enabled": self.tor_enabled,
+                "socks5_host": self.tor_socks5_host,
+                "socks5_port": self.tor_socks5_port,
+                "control_port": self.tor_control_port,
+            },
+            "rotation": {
+                "user_agent": self.rotation_user_agent,
+                "ip": self.ip_rotation,
+                "timing": self.rotation_timing,
+                "delay_range": list(self.rotation_delay_range),
+            },
+            "stealth": {
+                "webdriver_override": self.webdriver_override,
+                "canvas_noise": self.canvas_noise,
+                "webrtc_disable": self.webrtc_disable,
+                "geolocation_spoof": self.geolocation_spoof,
+            },
+            "retry": {
+                "max_attempts": self.max_retries,
+                "min_quality": self.min_quality,
+                "backoff": self.retry_backoff,
+            },
+            "output": {
+                "default_format": self.default_format,
+                "max_chars": self.max_chars,
+                "include_links": self.include_links,
+            },
+            "rate_limit": {
+                "rpm": self.rate_limit_rpm,
+            },
+        }
+
+    def update(self, updates: dict[str, Any]) -> dict[str, Any]:
+        """Apply nested runtime config updates and persist to disk."""
+        self._apply_nested_updates(updates)
+        self._persist()
+        return {"status": "ok", "config": self.to_api_dict()}
+
+    def _persist(self) -> None:
+        """Write the current config to the configured JSON file."""
+        path = self.config_file
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(self.to_api_dict(), f, indent=2)
+        except OSError:
+            pass
+
+    def enable_tor(self) -> dict[str, Any]:
+        """Convenience helper used by the /v1/config/profile/tor endpoint."""
+        return self.update(
+            {
+                "tor": {"enabled": True},
+                "proxy": {"enabled": True, "url": "socks5://tor:9050"},
+            }
+        )
+
+
+# Public singleton. Modules should import this directly.
 settings = PrySettings()

structure_monitor.py

@@ -149,6 +149,7 @@ async def monitor_page_structure(
                     .first()
                 )
 
+            previous_selectors: list[str]
             if previous_snap:
                 previous_selectors = previous_snap.selectors or []
                 previous_all_matched = previous_snap.all_matched

tests/test_api_integration.py

@@ -0,0 +1,406 @@
+"""HTTP integration tests for all Pry API routers.
+
+Tests that endpoints respond with correct status codes, response shapes,
+error handling, CORS headers, and auth gating — all via TestClient with
+mocked dependencies so no real scraping occurs.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+from __future__ import annotations
+
+from collections.abc import Iterator
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+# ── Fixtures ─────────────────────────────────────────────────────
+
+
+@pytest.fixture(autouse=True)
+def _mock_all_deps() -> Iterator[None]:
+    """Mock all external deps so no real scraping/parsing occurs."""
+    scraper_mock = MagicMock()
+    scraper_mock.scrape = AsyncMock(
+        return_value={
+            "status": "ok",
+            "url": "https://example.com",
+            "content": "# Mocked content",
+            "raw_html": "<html><body>Mocked</body></html>",
+            "method": "direct",
+        }
+    )
+    scraper_mock.crawl = AsyncMock(return_value=[{"url": "https://example.com", "status": "ok"}])
+    scraper_mock.map_urls = AsyncMock(
+        return_value=[{"url": "https://example.com/page1"}, {"url": "https://example.com/page2"}]
+    )
+    scraper_mock.detect_block = AsyncMock(
+        return_value={
+            "detected": True,
+            "type": "cloudflare",
+            "confidence": 0.95,
+        }
+    )
+
+    extractor_mock = MagicMock()
+    extractor_mock.extract = AsyncMock(
+        return_value={
+            "success": True,
+            "data": {"product_name": "Widget Pro", "price": "$29.99"},
+        }
+    )
+
+    parser_mock = MagicMock()
+    parser_mock.parse = AsyncMock(
+        return_value={
+            "success": True,
+            "data": {"title": "Test", "body": "Content"},
+        }
+    )
+
+    cache_mock = MagicMock()
+    cache_mock.get = MagicMock(return_value=None)
+    cache_mock.set = MagicMock()
+
+    advanced_mock = MagicMock()
+    advanced_mock.extract = AsyncMock(
+        return_value={
+            "success": True,
+            "data": {"name": "Widget Pro", "price": 29.99},
+        }
+    )
+
+    with patch.multiple(
+        "deps",
+        scraper=scraper_mock,
+        extractor=extractor_mock,
+        parser=parser_mock,
+        cache=cache_mock,
+        advanced=advanced_mock,
+    ):
+        yield
+
+
+@pytest.fixture
+def client() -> Iterator[TestClient]:
+    """TestClient with x402 middleware bypassed (it would require payment)."""
+    from api import app
+
+    with (
+        patch(
+            "x402_middleware.X402Middleware.__call__",
+            lambda self, scope, receive, send: self.app(scope, receive, send),
+        ),
+        TestClient(app) as c,
+    ):
+        yield c
+
+
+@pytest.fixture
+def client_auth(client: TestClient, monkeypatch) -> TestClient:
+    """TestClient with PRY_API_KEY set (auth required)."""
+    monkeypatch.setattr("api.settings.api_key", "test-api-key")
+    return client
+
+
+# ── Health & Readiness ───────────────────────────────────────────
+
+
+class TestHealthEndpoints:
+    """Test /health, /live, /ready — should work without auth."""
+
+    def test_health_returns_200(self, client: TestClient) -> None:
+        """Health returns 503 when deps unavailable (normal in test)."""
+        resp = client.get("/health")
+        assert resp.status_code in (200, 503)
+        body = resp.json()
+        assert "status" in body
+
+    def test_live_returns_200(self, client: TestClient) -> None:
+        resp = client.get("/live")
+        assert resp.status_code == 200
+        assert resp.json() == {"status": "alive"}
+
+    def test_ready_returns_200(self, client: TestClient) -> None:
+        resp = client.get("/ready")
+        assert resp.status_code == 200
+        assert resp.json() == {"status": "ready"}
+
+
+# ── OpenAPI / Docs ───────────────────────────────────────────────
+
+
+class TestDocsEndpoints:
+    """Docs and OpenAPI schema should be accessible without auth."""
+
+    def test_openapi_json(self, client: TestClient) -> None:
+        resp = client.get("/openapi.json")
+        assert resp.status_code == 200
+        schema = resp.json()
+        assert "openapi" in schema
+        assert "paths" in schema
+        assert len(schema["paths"]) > 0
+
+    def test_docs_redirects(self, client: TestClient) -> None:
+        resp = client.get("/docs")
+        assert resp.status_code == 200
+        assert "text/html" in resp.headers["content-type"]
+
+
+# ── Scraping Endpoints ───────────────────────────────────────────
+
+
+class TestScrapeEndpoint:
+    """POST /v1/scrape — core scraping endpoint."""
+
+    def test_scrape_success(self, client: TestClient) -> None:
+        resp = client.post("/v1/scrape", json={"url": "https://example.com"})
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body.get("success") is True
+        assert "data" in body
+        assert "markdown" in body["data"]
+        assert "metadata" in body["data"]
+
+    def test_scrape_missing_url_returns_422(self, client: TestClient) -> None:
+        resp = client.post("/v1/scrape", json={})
+        assert resp.status_code == 422
+        body = resp.json()
+        assert "detail" in body
+
+    def test_scrape_invalid_url_returns_422(self, client: TestClient) -> None:
+        """URL validation happens inside scraper, returns 200 with error."""
+        resp = client.post("/v1/scrape", json={"url": "not-a-url"})
+        # Scraper handles validation internally; response may be 200 with error body
+        assert resp.status_code in (200, 422)
+
+    def test_scrape_with_options(self, client: TestClient) -> None:
+        resp = client.post(
+            "/v1/scrape",
+            json={
+                "url": "https://example.com",
+                "timeout": 10,
+                "output_format": "markdown",
+            },
+        )
+        assert resp.status_code == 200
+
+    def test_scrape_response_shape(self, client: TestClient) -> None:
+        resp = client.post("/v1/scrape", json={"url": "https://example.com"})
+        assert resp.status_code == 200
+        body = resp.json()
+        # Success response should have data, not error
+        assert "error" not in body or not body.get("error")
+
+
+class TestDetectBlockEndpoint:
+    """POST /v1/detect-block — anti-bot detection."""
+
+    def test_detect_block_success(self, client: TestClient) -> None:
+        """detect-block takes raw string body, makes real HTTP calls."""
+        # This endpoint bypasses the scraper mock and makes real HTTP calls.
+        # In tests, it will either fail or return a result depending on network.
+        # We just verify it doesn't 500.
+        resp = client.post(
+            "/v1/detect-block",
+            data="https://example.com",
+            headers={"Content-Type": "application/json"},
+        )
+        assert resp.status_code in (200, 422, 503)
+
+
+class TestCrawlEndpoint:
+    """POST /v1/crawl — multi-page crawl."""
+
+    def test_crawl_success(self, client: TestClient) -> None:
+        resp = client.post(
+            "/v1/crawl",
+            json={"url": "https://example.com", "max_pages": 3},
+        )
+        assert resp.status_code == 200
+        body = resp.json()
+        # Crawl returns a dict with job info
+        assert isinstance(body, dict)
+
+
+class TestMapEndpoint:
+    """POST /v1/map — URL discovery."""
+
+    def test_map_success(self, client: TestClient) -> None:
+        resp = client.post("/v1/map", json={"url": "https://example.com"})
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body.get("success") is True
+        assert "links" in body.get("data", {})
+
+
+# ── Extraction Endpoints ─────────────────────────────────────────
+
+
+class TestExtractEndpoint:
+    """POST /v1/extract — structured data extraction."""
+
+    def test_extract_success(self, client: TestClient) -> None:
+        resp = client.post(
+            "/v1/extract",
+            json={
+                "url": "https://example.com",
+                "schema": {"product_name": "name of product"},
+            },
+        )
+        assert resp.status_code == 200
+        body = resp.json()
+        # Structure depends on router, but should have data
+        assert isinstance(body, dict)
+
+    def test_extract_missing_schema_returns_422(self, client: TestClient) -> None:
+        """Extract with just URL and no schema may still pass validation."""
+        resp = client.post("/v1/extract", json={"url": "https://example.com"})
+        # Schema may be optional; just verify it doesn't 500
+        assert resp.status_code in (200, 422)
+
+
+# ── Config Endpoints ─────────────────────────────────────────────
+
+
+class TestConfigEndpoint:
+    """GET /v1/config — runtime configuration."""
+
+    def test_config_success(self, client: TestClient) -> None:
+        resp = client.get("/v1/config")
+        assert resp.status_code == 200
+        body = resp.json()
+        assert isinstance(body, dict)
+
+
+# ── Stats Endpoints ──────────────────────────────────────────────
+
+
+class TestStatsEndpoint:
+    """GET /v1/stats — usage statistics."""
+
+    def test_stats_success(self, client: TestClient) -> None:
+        resp = client.get("/v0/stats")
+        assert resp.status_code == 200
+        body = resp.json()
+        assert isinstance(body, dict)
+
+
+# ── Templates Endpoints ──────────────────────────────────────────
+
+
+class TestTemplatesEndpoint:
+    """Template listing and retrieval."""
+
+    def test_list_templates(self, client: TestClient) -> None:
+        resp = client.get("/v1/templates")
+        # 404 or 200 depending on whether template listing exists
+        assert resp.status_code in (200, 404)
+
+
+# ── Error Handling ───────────────────────────────────────────────
+
+
+class TestErrorHandling:
+    """API returns consistent error shapes for HTTP errors."""
+
+    def test_404_returns_json(self, client: TestClient) -> None:
+        resp = client.get("/nonexistent-route-12345")
+        assert resp.status_code == 404
+        body = resp.json()
+        assert "detail" in body
+
+    def test_405_on_wrong_method(self, client: TestClient) -> None:
+        resp = client.get("/v1/scrape")
+        assert resp.status_code == 405
+
+    def test_422_on_invalid_json(self, client: TestClient) -> None:
+        resp = client.post(
+            "/v1/scrape", data="not-json", headers={"Content-Type": "application/json"}
+        )
+        assert resp.status_code == 422
+
+
+# ── Auth Gating ──────────────────────────────────────────────────
+
+
+class TestAuth:
+    """PRY_API_KEY gating on protected endpoints."""
+
+    def test_health_is_public(self, client_auth: TestClient) -> None:
+        """Health endpoints should not require auth."""
+        resp = client_auth.get("/health")
+        assert resp.status_code in (200, 503)
+
+    def test_scrape_requires_auth(self, client_auth: TestClient) -> None:
+        """Scraping endpoints require valid Bearer token when api_key is set."""
+        resp = client_auth.post(
+            "/v1/scrape",
+            json={"url": "https://example.com"},
+        )
+        assert resp.status_code == 401
+
+    def test_scrape_with_valid_auth(self, client_auth: TestClient) -> None:
+        """Valid Bearer token should pass auth."""
+        resp = client_auth.post(
+            "/v1/scrape",
+            json={"url": "https://example.com"},
+            headers={"authorization": "Bearer test-api-key"},
+        )
+        assert resp.status_code == 200
+
+    def test_scrape_with_wrong_auth(self, client_auth: TestClient) -> None:
+        """Wrong Bearer token should get 401."""
+        resp = client_auth.post(
+            "/v1/scrape",
+            json={"url": "https://example.com"},
+            headers={"authorization": "Bearer wrong-key"},
+        )
+        assert resp.status_code == 401
+
+
+# ── CORS Headers ─────────────────────────────────────────────────
+
+
+class TestCORS:
+    """CORS middleware should set broad allow-origin."""
+
+    def test_cors_headers_present(self, client: TestClient) -> None:
+        resp = client.options(
+            "/v1/scrape",
+            headers={
+                "Origin": "https://example.com",
+                "Access-Control-Request-Method": "POST",
+            },
+        )
+        assert resp.status_code == 200
+        headers = resp.headers
+        assert headers.get("access-control-allow-origin") == "*"
+        assert "POST" in headers.get("access-control-allow-methods", "")
+
+
+class TestMetricsEndpoint:
+    """GET /metrics — Prometheus metrics endpoint."""
+
+    def test_metrics_returns_200(self, client: TestClient) -> None:
+        resp = client.get("/metrics")
+        assert resp.status_code == 200
+        assert "text/plain" in resp.headers["content-type"]
+        body = resp.text
+        # Should contain at least some metric names
+        assert "pry_" in body or "# HELP" in body
+
+    def test_metrics_contains_request_count(self, client: TestClient) -> None:
+        """After one scrape request, request count metric should exist."""
+        # Trigger a request first
+        client.post("/v1/scrape", json={"url": "https://example.com"})
+        resp = client.get("/metrics")
+        assert resp.status_code == 200
+        # The metric may or may not have been recorded depending on the
+        # middleware (it might not use prometheus_client.Counter).
+        # Just verify the endpoint works.
+        assert len(resp.text) > 0

tests/test_api_mcp.py

@@ -0,0 +1,114 @@
+"""HTTP integration tests for MCP endpoints (mounted at /mcp).
+
+Tests the JSON-RPC endpoint, health, SSE transport, and message posting.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+from __future__ import annotations
+
+from collections.abc import Iterator
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+
+@pytest.fixture
+def client() -> Iterator[TestClient]:
+    """TestClient with x402 middleware bypassed."""
+    from api import app
+
+    with (
+        patch(
+            "x402_middleware.X402Middleware.__call__",
+            lambda self, scope, receive, send: self.app(scope, receive, send),
+        ),
+        TestClient(app) as c,
+    ):
+        yield c
+
+
+@pytest.fixture
+def mock_mcp_server() -> Iterator[None]:
+    """Mock the MCP server to return controlled responses."""
+    mock_server = AsyncMock()
+    mock_server.handle_request = AsyncMock(
+        return_value={
+            "jsonrpc": "2.0",
+            "id": 1,
+            "result": {
+                "tools": [
+                    {
+                        "name": "pry_scrape",
+                        "description": "Scrape a URL",
+                        "inputSchema": {"type": "object", "properties": {}},
+                    }
+                ]
+            },
+        }
+    )
+    mock_server.tools = [{"name": "pry_scrape"}]
+    mock_server.resources = []
+    mock_server.prompts = []
+
+    # The MCP server is lazily created inside api._get_mcp_server().
+    # We patch the function to return our mock.
+    patcher = patch("api._get_mcp_server", return_value=mock_server)
+    patcher.start()
+    yield
+    patcher.stop()
+
+
+class TestMCPHealth:
+    """GET /mcp/health — MCP server health."""
+
+    def test_mcp_health_returns_200(self, client: TestClient, mock_mcp_server: None) -> None:
+        resp = client.get("/mcp/health")
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body["status"] == "ok"
+        assert body["server"] == "pry-mcp"
+        assert "tools_count" in body
+
+
+class TestMCPJsonRPC:
+    """POST /mcp/ — MCP JSON-RPC endpoint."""
+
+    def test_list_tools_request(self, client: TestClient, mock_mcp_server: None) -> None:
+        resp = client.post(
+            "/mcp/",
+            json={
+                "jsonrpc": "2.0",
+                "id": 1,
+                "method": "tools/list",
+                "params": {},
+            },
+        )
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body.get("jsonrpc") == "2.0"
+        assert "result" in body
+
+    def test_invalid_json_rpc_returns_error(
+        self, client: TestClient, mock_mcp_server: None
+    ) -> None:
+        resp = client.post(
+            "/mcp/",
+            json={"not": "valid json-rpc"},
+        )
+        # Should still respond with valid JSON
+        assert resp.status_code == 200
+
+
+class TestMCPSSE:
+    """GET /mcp/sse — MCP SSE transport endpoint."""
+
+    @pytest.mark.skip(reason="SSE is streaming; would hang in test")
+    def test_sse_endpoint_returns_streaming(self, client: TestClient) -> None:
+        resp = client.get("/mcp/sse")
+        assert resp.status_code == 200
+        assert resp.headers.get("content-type", "").startswith("text/event-stream")

tests/test_api_surface_snapshot.py

@@ -0,0 +1,84 @@
+"""Snapshot tests for the public MCP and x402 API surfaces.
+
+These tests guard against accidental breakage when refactoring
+mcp_production.py and x402.py into sub-packages.
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+
+class TestMCPPublicSurface:
+    """The mcp_production shim must expose the same public API as before."""
+
+    def test_required_symbols_exist(self) -> None:
+        import mcp_production
+
+        required = {
+            "DEFAULT_BASE_URL",
+            "MCP_PROTOCOL_VERSION",
+            "SERVER_NAME",
+            "SERVER_VERSION",
+            "MCPLoggingHandler",
+            "PRY_PROMPTS",
+            "PRY_RESOURCES",
+            "PRY_TOOLS",
+            "create_server",
+            "main",
+            "make_fallback_server",
+            "notify_resource_changed",
+            "notify_resource_list_changed",
+            "notify_tool_list_changed",
+            "register_all",
+            "register_mcp_notification_observer",
+            "set_active_mcp_server",
+            "unregister_mcp_notification_observer",
+        }
+        missing = {name for name in required if not hasattr(mcp_production, name)}
+        assert not missing, f"Missing public MCP symbols: {missing}"
+
+    def test_tool_definitions_are_non_empty(self) -> None:
+        import mcp_production
+
+        assert isinstance(mcp_production.PRY_TOOLS, list)
+        assert len(mcp_production.PRY_TOOLS) > 0
+        assert isinstance(mcp_production.PRY_RESOURCES, list)
+        assert isinstance(mcp_production.PRY_PROMPTS, list)
+
+
+class TestX402PublicSurface:
+    """The x402 shim must expose the same public API as before."""
+
+    def test_required_symbols_exist(self) -> None:
+        import x402
+
+        required = {
+            "X402Asset",
+            "X402Handler",
+            "X402Network",
+            "X402Scheme",
+            "FacilitatorConfig",
+            "FacilitatorRouter",
+            "build_payment_required",
+            "create_batch_payment",
+            "create_payment_request",
+            "get_facilitator_router",
+            "is_batch_paid",
+            "require_payment_legacy",
+            "set_facilitator_router",
+            "verify_batch_payment",
+        }
+        missing = {name for name in required if not hasattr(x402, name)}
+        assert not missing, f"Missing public x402 symbols: {missing}"
+
+    def test_enum_values_consistent(self) -> None:
+        import x402
+
+        assert x402.X402Scheme.EXACT.value == "exact"
+        assert x402.X402Network.ETHEREUM.value == "ethereum"
+        assert x402.X402Asset.USDC.value == "USDC"

tests/test_fallback_integration.py

@@ -0,0 +1,266 @@
+"""Integration tests for the full fallback chain: retry + circuit breaker + tier dispatch."""
+
+from __future__ import annotations
+
+import time as time_module
+from contextlib import ExitStack
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+import resilience
+import ultimate_scraper
+from resilience import ResilienceRegistry
+from retry import Jitter, RetryConfig, async_retry
+
+
+def _valid_html(text: str = "content") -> str:
+    body = f"<p>{text}</p>" * 40
+    return (
+        "<!DOCTYPE html><html lang=\"en\"><head><title>T</title>"
+        "<meta charset=\"utf-8\"></head>"
+        f"<body>{body}</body></html>"
+    )
+
+
+_TIER_NAMES = [
+    "_tier_direct",
+    "_tier_tls_fingerprint",
+    "_tier_cloudscraper",
+    "_tier_flaresolverr",
+    "_tier_undetected",
+    "_tier_camoufox",
+    "_tier_playwright",
+    "_tier_googlebot",
+    "_tier_premium_proxy",
+    "_tier_archive",
+    "_tier_google_cache",
+    "_tier_tor",
+]
+
+
+@pytest.fixture
+def fresh_registry(monkeypatch):
+    """Clean ResilienceRegistry patched into both modules."""
+    reg = ResilienceRegistry()
+    monkeypatch.setattr(resilience, "registry", reg)
+    monkeypatch.setattr(ultimate_scraper, "registry", reg)
+    return reg
+
+
+@pytest.fixture
+def mock_all_tiers():
+    """Return a dict of AsyncMocks for every tier method."""
+    return {name: AsyncMock(return_value=None) for name in _TIER_NAMES}
+
+
+@pytest.fixture
+def scraper_with_mocks(mock_all_tiers):
+    """UltimateScraper with all tier methods mocked."""
+    s = ultimate_scraper.UltimateScraper()
+    with ExitStack() as stack:
+        for name in _TIER_NAMES:
+            stack.enter_context(patch.object(s, name, new=mock_all_tiers[name]))
+        yield s, mock_all_tiers
+
+
+BASE_CFG = RetryConfig(max_retries=2, base_delay=0.01, jitter=Jitter.NONE)
+BASE_CFG_CB = RetryConfig(max_retries=2, base_delay=0.01, jitter=Jitter.NONE)
+
+
+class TestRetryWithScraper:
+    """Retry wrapping scraper entry points."""
+
+    @pytest.mark.anyio
+    async def test_retry_with_exception_wrapper(self):
+        """Retry wraps a flaky function that raises then succeeds."""
+        call_count = 0
+
+        async def flaky_fn() -> str:
+            nonlocal call_count
+            call_count += 1
+            if call_count == 1:
+                raise ConnectionError("first attempt failed")
+            return "success"
+
+        result = await async_retry(flaky_fn, config=BASE_CFG)
+        assert result == "success"
+        assert call_count == 2
+
+    @pytest.mark.anyio
+    async def test_retry_exhausts_and_raises(self, scraper_with_mocks):
+        """Retry exhausts attempts and re-raises."""
+        s, _mocks = scraper_with_mocks
+
+        async def scrape_or_raise(url: str) -> dict:
+            result = await s.scrape(url)
+            if result.get("status") == "error":
+                raise RuntimeError(result.get("error", "scrape failed"))
+            return result
+
+        with pytest.raises(RuntimeError, match=r"scrape failed|_failed|circuit_open"):
+            await async_retry(scrape_or_raise, "https://example.com/page", config=BASE_CFG)
+
+
+class TestCircuitBreakerIntegration:
+    """Circuit breaker with retry and tier skipping."""
+
+    @pytest.mark.anyio
+    async def test_retry_skips_open_breaker(self, fresh_registry):
+        """Open circuit breaker causes retry to skip immediately."""
+        name = "cb_int_block"
+        cb = fresh_registry.breaker(name)
+        cb.failure_threshold = 1
+        fresh_registry.record_service_result(name, False)
+        assert fresh_registry.is_service_allowed(name) is False
+
+        fn = AsyncMock(side_effect=ConnectionError("down"))
+        cfg = RetryConfig(max_retries=3, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE)
+
+        with pytest.raises(RuntimeError, match="Circuit breaker"):
+            await async_retry(fn, config=cfg)
+        assert fn.call_count == 0
+        del resilience.registry.breakers[name]
+
+    @pytest.mark.anyio
+    async def test_circuit_records_success_on_retry(self, fresh_registry):
+        """Successful retry records success to circuit breaker."""
+        name = "cb_int_success"
+        fn = AsyncMock(return_value="ok")
+        cfg = RetryConfig(max_retries=1, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE)
+        result = await async_retry(fn, config=cfg)
+        assert result == "ok"
+        cb = fresh_registry.breaker(name)
+        assert cb.successes >= 1
+        del resilience.registry.breakers[name]
+
+    @pytest.mark.anyio
+    async def test_recovery_after_half_open(self, fresh_registry, monkeypatch):
+        """After recovery window, retry should attempt and succeed."""
+        now = 1_700_000_000.0
+        monkeypatch.setattr(time_module, "time", lambda: now)
+        monkeypatch.setattr(resilience.time, "time", lambda: now)
+
+        cb = fresh_registry.breaker("ollama_int")
+        cb.failure_threshold = 1
+        cb.recovery_seconds = 30
+        fresh_registry.record_service_result("ollama_int", False)
+        assert cb.state == "open"
+
+        future = now + 31
+        monkeypatch.setattr(time_module, "time", lambda: future)
+        monkeypatch.setattr(resilience.time, "time", lambda: future)
+
+        fn = AsyncMock(return_value="ok")
+        cfg = RetryConfig(max_retries=1, circuit_breaker="ollama_int", base_delay=0.01, jitter=Jitter.NONE)
+        result = await async_retry(fn, config=cfg)
+        assert result == "ok"
+
+    @pytest.mark.anyio
+    async def test_skip_open_breaker_tier(self, fresh_registry, mock_all_tiers):
+        """Breaker open for flaresolverr skips that tier."""
+        s = ultimate_scraper.UltimateScraper()
+        html = _valid_html("archive")
+        mock_all_tiers["_tier_archive"].return_value = html
+
+        fresh_registry.breaker("flaresolverr").failure_threshold = 1
+        fresh_registry.record_service_result("flaresolverr", False)
+
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(s, name, new=mock_all_tiers[name]))
+            result = await s.scrape("https://example.com/page")
+        assert result["status"] == "ok"
+        assert result["method"] == "archive"
+        mock_all_tiers["_tier_flaresolverr"].assert_not_awaited()
+
+    @pytest.mark.anyio
+    async def test_multiple_open_breakers(self, fresh_registry, mock_all_tiers):
+        """Multiple open breakers skip their tiers."""
+        s = ultimate_scraper.UltimateScraper()
+        html = _valid_html("tor finish")
+        mock_all_tiers["_tier_tor"].return_value = html
+
+        for name in ["flaresolverr", "playwright"]:
+            cb = fresh_registry.breaker(name)
+            cb.failure_threshold = 1
+            fresh_registry.record_service_result(name, False)
+
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(s, name, new=mock_all_tiers[name]))
+            result = await s.scrape("https://example.com/page")
+        assert result["status"] == "ok"
+        mock_all_tiers["_tier_flaresolverr"].assert_not_awaited()
+        mock_all_tiers["_tier_playwright"].assert_not_awaited()
+
+
+class TestDomainTierMemoryWithRetry:
+    """Domain tier memory + retry combined."""
+
+    @pytest.mark.anyio
+    async def test_domain_memory_used_on_retry(self, fresh_registry, scraper_with_mocks):
+        """After domain learns best tier, prefers it."""
+        s, _mocks = scraper_with_mocks
+        fresh_registry.record_domain_tier_result("example.com", "archive", success=True, latency=0.1)
+        html = _valid_html("archive content")
+        _mocks["_tier_archive"].return_value = html
+        _mocks["_tier_direct"].return_value = _valid_html("direct")
+
+        result = await s.scrape("https://example.com/page")
+        assert result["status"] == "ok"
+        _mocks["_tier_archive"].assert_awaited()
+        _mocks["_tier_direct"].assert_not_awaited()
+
+
+class TestGracefulDegradation:
+    """End-to-end graceful degradation."""
+
+    @pytest.mark.anyio
+    async def test_all_tiers_fail_returns_error(self, fresh_registry, mock_all_tiers):
+        """All tiers return None, scrape returns error."""
+        s = ultimate_scraper.UltimateScraper()
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(s, name, new=mock_all_tiers[name]))
+            result = await s.scrape("https://example.com/page")
+        assert result["status"] == "error"
+
+    @pytest.mark.anyio
+    async def test_circuit_breaker_with_error_dict(self, fresh_registry, mock_all_tiers):
+        """If scrape returns error (not exception), circuit breaker still records failure."""
+        s = ultimate_scraper.UltimateScraper()
+        name = "cb_degradation"
+
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(s, name, new=mock_all_tiers[name]))
+
+            async def scrape_and_raise(url: str) -> dict:
+                result = await s.scrape(url)
+                if result.get("status") == "error":
+                    raise RuntimeError(result.get("error", "scrape failed"))
+                return result
+
+            cfg = RetryConfig(max_retries=1, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE)
+            with pytest.raises(RuntimeError):
+                await async_retry(scrape_and_raise, "https://example.com/page", config=cfg)
+
+        if name in resilience.registry.breakers:
+            del resilience.registry.breakers[name]
+
+
+class TestRetrySettings:
+    """Retry module integration with settings."""
+
+    def test_retry_config_from_settings(self) -> None:
+        from settings import PrySettings
+        settings = PrySettings()
+        assert settings.retry_backoff == "exponential"
+
+    def test_api_dict_includes_backoff(self) -> None:
+        from settings import PrySettings
+        settings = PrySettings()
+        api_dict = settings.to_api_dict()
+        assert "retry" in api_dict
+        assert api_dict["retry"]["backoff"] == "exponential"

tests/test_mconfig.py

@@ -1,47 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 Rug Munch Media LLC
-#
-# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
-# Licensed under MIT. See LICENSE.
-"""Tests for Pry config."""
-
-from mconfig import PryConfig
-
-
-def test_config_get_default() -> None:
-    c = PryConfig()
-    assert c.get("nonexistent.key", "fallback") == "fallback"
-
-
-def test_config_get_retry_exists() -> None:
-    c = PryConfig()
-    val = c.get("retry.max_attempts")
-    assert isinstance(val, int)
-    assert val > 0
-
-
-def test_config_get_rate_limit_exists() -> None:
-    c = PryConfig()
-    val = c.get("rate_limit.rpm")
-    assert isinstance(val, int)
-    assert val > 0
-
-
-def test_config_to_dict_excludes_private() -> None:
-    c = PryConfig()
-    d = c.to_dict()
-    assert "_proxy_chain" not in d
-    assert "_proxy_url" not in d
-
-
-def test_config_update_returns_ok() -> None:
-    c = PryConfig()
-    result = c.update({"retry": {"max_attempts": 5}})
-    assert result["status"] == "ok"
-    assert c.get("retry.max_attempts") == 5
-
-
-def test_config_get_proxy_url_string_or_none() -> None:
-    c = PryConfig()
-    url = c.get_proxy_url()
-    assert url is None or isinstance(url, str)

tests/test_mcp_production.py

@@ -7,7 +7,7 @@
 
 import asyncio
 
-from mcp_production import MCP_PROTOCOL_VERSION, PRY_TOOLS, make_fallback_server, register_all
+from pry_mcp import MCP_PROTOCOL_VERSION, PRY_TOOLS, make_fallback_server, register_all
 
 
 def test_mcp_protocol_version() -> None:

tests/test_resilience.py

@@ -0,0 +1,219 @@
+"""Tests for resilience primitives and their integration into UltimateScraper."""
+
+from __future__ import annotations
+
+import time as time_module
+from contextlib import ExitStack
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+import resilience
+import ultimate_scraper
+from resilience import CircuitBreaker, CircuitState, DomainTierMemory, ResilienceRegistry
+
+
+def _valid_html(text: str) -> str:
+    body = f"<p>{text}</p>" * 40
+    return (
+        "<!DOCTYPE html><html lang=\"en\"><head><title>T</title>"
+        "<meta charset=\"utf-8\"></head>"
+        f"<body>{body}</body></html>"
+    )
+
+
+@pytest.fixture
+def fresh_registry(monkeypatch):
+    """Provide a clean ResilienceRegistry and patch it into UltimateScraper."""
+    reg = ResilienceRegistry()
+    monkeypatch.setattr(resilience, "registry", reg)
+    monkeypatch.setattr(ultimate_scraper, "registry", reg)
+    return reg
+
+
+@pytest.fixture
+def frozen_time(monkeypatch):
+    """Freeze time.time() for deterministic circuit-breaker tests."""
+    now = 1_700_000_000.0
+
+    def fake_time():
+        return now
+
+    monkeypatch.setattr(time_module, "time", fake_time)
+    monkeypatch.setattr(resilience.time, "time", fake_time)
+    monkeypatch.setattr(ultimate_scraper.time, "time", fake_time)
+    return lambda offset=0: setattr(fake_time, "__wrapped__", None) or None
+
+
+class TestCircuitBreaker:
+    def test_starts_closed_and_allows(self):
+        cb = CircuitBreaker(name="test")
+        assert cb.state == CircuitState.CLOSED
+        assert cb.is_allowed() is True
+
+    def test_opens_after_threshold_failures(self):
+        cb = CircuitBreaker(name="test", failure_threshold=3, recovery_seconds=30)
+        cb.record_failure()
+        cb.record_failure()
+        assert cb.is_allowed() is True
+        cb.record_failure()
+        assert cb.state == CircuitState.OPEN
+        assert cb.is_allowed() is False
+
+    def test_half_open_after_recovery(self, frozen_time, monkeypatch):
+        cb = CircuitBreaker(name="test", failure_threshold=1, recovery_seconds=60)
+        cb.record_failure()
+        assert cb.state == CircuitState.OPEN
+        # Advance time past recovery window.
+        now = 1_700_000_000.0 + 61
+        monkeypatch.setattr(time_module, "time", lambda: now)
+        monkeypatch.setattr(resilience.time, "time", lambda: now)
+        assert cb.is_allowed() is True
+        assert cb.state == CircuitState.HALF_OPEN
+
+    def test_closes_after_half_open_successes(self, frozen_time, monkeypatch):
+        cb = CircuitBreaker(
+            name="test", failure_threshold=1, recovery_seconds=60, half_open_max_tries=2
+        )
+        cb.record_failure()
+        now = 1_700_000_000.0 + 61
+        monkeypatch.setattr(time_module, "time", lambda: now)
+        monkeypatch.setattr(resilience.time, "time", lambda: now)
+        assert cb.is_allowed() is True
+        cb.record_success()
+        assert cb.state == CircuitState.HALF_OPEN
+        assert cb.is_allowed() is True
+        cb.record_success()
+        assert cb.state == CircuitState.CLOSED
+
+    def test_failure_in_half_open_reopens(self, frozen_time, monkeypatch):
+        cb = CircuitBreaker(name="test", failure_threshold=1, recovery_seconds=60)
+        cb.record_failure()
+        now = 1_700_000_000.0 + 61
+        monkeypatch.setattr(time_module, "time", lambda: now)
+        monkeypatch.setattr(resilience.time, "time", lambda: now)
+        cb.is_allowed()
+        cb.record_failure()
+        assert cb.state == CircuitState.OPEN
+
+
+class TestDomainTierMemory:
+    def test_picks_best_tier(self):
+        mem = DomainTierMemory()
+        mem.record("example.com", "direct", success=True, latency=0.1)
+        mem.record("example.com", "direct", success=True, latency=0.1)
+        mem.record("example.com", "playwright", success=False, latency=0.1)
+        best = mem.best_tier("example.com", ["direct", "playwright"])
+        assert best == "direct"
+
+    def test_ignores_recent_failures(self, frozen_time, monkeypatch):
+        mem = DomainTierMemory()
+        now = 1_700_000_000.0
+        monkeypatch.setattr(time_module, "time", lambda: now)
+        mem.record("example.com", "direct", success=True, latency=0.1)
+        mem.record("example.com", "direct", success=False, latency=0.1)
+        assert mem.best_tier("example.com", ["direct"]) is None
+
+    def test_returns_none_for_unknown_domain(self):
+        mem = DomainTierMemory()
+        assert mem.best_tier("example.com", ["direct"]) is None
+
+
+class TestResilienceRegistry:
+    def test_stats_shape(self, fresh_registry):
+        stats = fresh_registry.stats()
+        assert "circuit_breakers" in stats
+        assert "domain_memory" in stats
+        assert set(stats["circuit_breakers"].keys()) == set(ResilienceRegistry.SERVICE_NAMES)
+
+    def test_service_result_toggles_breaker(self, fresh_registry):
+        fresh_registry.breaker("flaresolverr").failure_threshold = 1
+        fresh_registry.record_service_result("flaresolverr", False)
+        assert fresh_registry.is_service_allowed("flaresolverr") is False
+        fresh_registry.record_service_result("flaresolverr", True)
+        # In half-open after recovery, not immediate.
+        assert fresh_registry.breaker("flaresolverr").state == CircuitState.OPEN
+
+
+_TIER_NAMES = [
+    "_tier_direct",
+    "_tier_tls_fingerprint",
+    "_tier_cloudscraper",
+    "_tier_flaresolverr",
+    "_tier_undetected",
+    "_tier_camoufox",
+    "_tier_playwright",
+    "_tier_googlebot",
+    "_tier_premium_proxy",
+    "_tier_archive",
+    "_tier_google_cache",
+    "_tier_tor",
+]
+
+
+class TestUltimateScraperResilience:
+    @pytest.mark.anyio
+    async def test_skips_circuit_open_tier(self, fresh_registry):
+        scraper = ultimate_scraper.UltimateScraper()
+        fresh_registry.breaker("flaresolverr").failure_threshold = 1
+        fresh_registry.record_service_result("flaresolverr", False)
+
+        html = _valid_html("archive content")
+        mocks = {
+            name: AsyncMock(return_value=None)
+            for name in _TIER_NAMES
+        }
+        mocks["_tier_archive"].return_value = html
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(scraper, name, new=mocks[name]))
+            result = await scraper.scrape("https://example.com/page")
+        assert result["status"] == "ok"
+        assert result["method"] == "archive"
+        mocks["_tier_direct"].assert_awaited_once()
+        mocks["_tier_cloudscraper"].assert_awaited_once()
+        mocks["_tier_flaresolverr"].assert_not_awaited()
+
+    @pytest.mark.anyio
+    async def test_prefers_domain_memory_best_tier(self, fresh_registry):
+        scraper = ultimate_scraper.UltimateScraper()
+        fresh_registry.record_domain_tier_result("example.com", "archive", success=True)
+        html = _valid_html("archived")
+        mocks = {name: AsyncMock(return_value=None) for name in _TIER_NAMES}
+        mocks["_tier_archive"].return_value = html
+        mocks["_tier_direct"].return_value = _valid_html("direct")
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(scraper, name, new=mocks[name]))
+            result = await scraper.scrape("https://example.com/page")
+        assert result["status"] == "ok"
+        assert result["method"] == "archive"
+        mocks["_tier_archive"].assert_awaited()
+        mocks["_tier_direct"].assert_not_awaited()
+
+    @pytest.mark.anyio
+    async def test_records_tier_results_after_success(self, fresh_registry):
+        scraper = ultimate_scraper.UltimateScraper()
+        html = _valid_html("direct content")
+        mocks = {name: AsyncMock(return_value=None) for name in _TIER_NAMES}
+        mocks["_tier_direct"].return_value = html
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(scraper, name, new=mocks[name]))
+            await scraper.scrape("https://example.com/page")
+        assert fresh_registry.best_tier_for_domain(
+            "example.com", ["direct", "archive"]
+        ) == "direct"
+
+    @pytest.mark.anyio
+    async def test_records_tier_failure_no_success_memory(self, fresh_registry):
+        scraper = ultimate_scraper.UltimateScraper()
+        mocks = {name: AsyncMock(return_value=None) for name in _TIER_NAMES}
+        with ExitStack() as stack:
+            for name in _TIER_NAMES:
+                stack.enter_context(patch.object(scraper, name, new=mocks[name]))
+            result = await scraper.scrape("https://example.com/page")
+        assert result["status"] == "error"
+        assert fresh_registry.best_tier_for_domain(
+            "example.com", ["direct", "archive"]
+        ) is None

tests/test_retry.py

@@ -0,0 +1,296 @@
+"""Tests for retry/backoff helpers."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+import resilience
+from retry import (
+    BackoffStrategy,
+    Jitter,
+    RetryConfig,
+    _apply_jitter,
+    _calculate_delay,
+    async_retry,
+    make_retry_decorator,
+)
+
+
+class TestBackoffStrategy:
+    def test_enum_values(self) -> None:
+        assert BackoffStrategy.EXPONENTIAL.value == "exponential"
+        assert BackoffStrategy.LINEAR.value == "linear"
+        assert BackoffStrategy.FIXED.value == "fixed"
+
+
+class TestJitter:
+    def test_enum_values(self) -> None:
+        assert Jitter.NONE.value == "none"
+        assert Jitter.FULL.value == "full"
+        assert Jitter.EQUAL.value == "equal"
+        assert Jitter.DECORRELATED.value == "decorrelated"
+
+
+class TestRetryConfig:
+    def test_defaults(self) -> None:
+        cfg = RetryConfig()
+        assert cfg.max_retries == 3
+        assert cfg.base_delay == 1.0
+        assert cfg.max_delay == 60.0
+        assert cfg.strategy == BackoffStrategy.EXPONENTIAL
+        assert cfg.jitter == Jitter.FULL
+        assert cfg.circuit_breaker is None
+
+    def test_custom_values(self) -> None:
+        cfg = RetryConfig(
+            max_retries=5,
+            base_delay=0.5,
+            max_delay=30.0,
+            strategy=BackoffStrategy.LINEAR,
+            jitter=Jitter.NONE,
+            circuit_breaker="flaresolverr",
+        )
+        assert cfg.max_retries == 5
+        assert cfg.base_delay == 0.5
+        assert cfg.max_delay == 30.0
+        assert cfg.strategy == BackoffStrategy.LINEAR
+        assert cfg.jitter == Jitter.NONE
+        assert cfg.circuit_breaker == "flaresolverr"
+
+
+class TestCalculateDelay:
+    def test_exponential_doubles(self) -> None:
+        cfg = RetryConfig(strategy=BackoffStrategy.EXPONENTIAL, jitter=Jitter.NONE)
+        assert _calculate_delay(0, cfg) == 1.0
+        assert _calculate_delay(1, cfg) == 2.0
+        assert _calculate_delay(2, cfg) == 4.0
+        assert _calculate_delay(3, cfg) == 8.0
+
+    def test_exponential_capped_at_max(self) -> None:
+        cfg = RetryConfig(
+            strategy=BackoffStrategy.EXPONENTIAL,
+            base_delay=1.0,
+            max_delay=5.0,
+            jitter=Jitter.NONE,
+        )
+        assert _calculate_delay(0, cfg) == 1.0
+        assert _calculate_delay(1, cfg) == 2.0
+        assert _calculate_delay(2, cfg) == 4.0
+        assert _calculate_delay(3, cfg) == 5.0
+
+    def test_linear_increases_by_base(self) -> None:
+        cfg = RetryConfig(strategy=BackoffStrategy.LINEAR, jitter=Jitter.NONE)
+        assert _calculate_delay(0, cfg) == 1.0
+        assert _calculate_delay(1, cfg) == 2.0
+        assert _calculate_delay(2, cfg) == 3.0
+
+    def test_fixed_always_base(self) -> None:
+        cfg = RetryConfig(strategy=BackoffStrategy.FIXED, jitter=Jitter.NONE)
+        assert _calculate_delay(0, cfg) == 1.0
+        assert _calculate_delay(5, cfg) == 1.0
+        assert _calculate_delay(99, cfg) == 1.0
+
+    def test_linear_capped(self) -> None:
+        cfg = RetryConfig(
+            strategy=BackoffStrategy.LINEAR,
+            base_delay=10.0,
+            max_delay=25.0,
+            jitter=Jitter.NONE,
+        )
+        assert _calculate_delay(0, cfg) == 10.0
+        assert _calculate_delay(1, cfg) == 20.0
+        assert _calculate_delay(2, cfg) == 25.0
+
+
+class TestJitterApplication:
+    def test_none_returns_delay(self) -> None:
+        assert _apply_jitter(5.0, Jitter.NONE) == 5.0
+
+    def test_full_is_between_0_and_delay(self) -> None:
+        for _ in range(50):
+            result = _apply_jitter(10.0, Jitter.FULL)
+            assert 0 <= result <= 10.0
+
+    def test_equal_is_positive(self) -> None:
+        for _ in range(50):
+            result = _apply_jitter(10.0, Jitter.EQUAL)
+            assert 5 <= result <= 10.0
+
+    def test_decorrelated_uses_previous(self) -> None:
+        for _ in range(50):
+            result = _apply_jitter(5.0, Jitter.DECORRELATED, previous_delay=2.0)
+            assert 2.0 <= result <= 6.0
+
+    def test_decorrelated_without_previous_returns_delay(self) -> None:
+        assert _apply_jitter(5.0, Jitter.DECORRELATED, previous_delay=0) == 5.0
+
+
+class TestAsyncRetry:
+    @pytest.mark.asyncio
+    async def test_success_on_first_try(self) -> None:
+        fn = AsyncMock(return_value="ok")
+        result = await async_retry(fn, config=RetryConfig(jitter=Jitter.NONE))
+        assert result == "ok"
+        assert fn.call_count == 1
+
+    @pytest.mark.asyncio
+    async def test_succeeds_after_retries(self) -> None:
+        fn = AsyncMock(side_effect=[ValueError("fail1"), ValueError("fail2"), "ok"])
+        result = await async_retry(
+            fn,
+            config=RetryConfig(max_retries=3, base_delay=0.01, jitter=Jitter.NONE),
+        )
+        assert result == "ok"
+        assert fn.call_count == 3
+
+    @pytest.mark.asyncio
+    async def test_exhausts_retries_and_raises(self) -> None:
+        fn = AsyncMock(side_effect=ValueError("always fails"))
+        with pytest.raises(ValueError, match="always fails"):
+            await async_retry(
+                fn,
+                config=RetryConfig(max_retries=2, base_delay=0.01, jitter=Jitter.NONE),
+            )
+        assert fn.call_count == 3
+
+    @pytest.mark.asyncio
+    async def test_non_retryable_exception_passthrough(self) -> None:
+        fn = AsyncMock(side_effect=TypeError("not retryable"))
+        cfg = RetryConfig(
+            retryable_exceptions=(ValueError,),
+            base_delay=0.01,
+            jitter=Jitter.NONE,
+        )
+        with pytest.raises(TypeError, match="not retryable"):
+            await async_retry(fn, config=cfg)
+        assert fn.call_count == 1
+
+    @pytest.mark.asyncio
+    async def test_retryable_exception_does_retry(self) -> None:
+        fn = AsyncMock(side_effect=[ValueError("retry"), "ok"])
+        cfg = RetryConfig(
+            max_retries=2,
+            retryable_exceptions=(ValueError,),
+            base_delay=0.01,
+            jitter=Jitter.NONE,
+        )
+        result = await async_retry(fn, config=cfg)
+        assert result == "ok"
+        assert fn.call_count == 2
+
+    @pytest.mark.asyncio
+    async def test_max_time_exceeded_stops_retrying(self) -> None:
+        fn = AsyncMock(side_effect=ValueError("fail"))
+        cfg = RetryConfig(
+            max_retries=5,
+            max_time=0.05,
+            base_delay=0.1,
+            jitter=Jitter.NONE,
+        )
+        with pytest.raises(ValueError):
+            await async_retry(fn, config=cfg)
+        assert fn.call_count < 6
+
+    @pytest.mark.asyncio
+    async def test_circuit_breaker_blocks_retry(self) -> None:
+        name = "cb_block_utest"
+        cb = resilience.registry.breaker(name)
+        cb.failure_threshold = 1
+        resilience.registry.record_service_result(name, False)
+        assert resilience.registry.is_service_allowed(name) is False
+
+        fn = AsyncMock(side_effect=ConnectionError("fail"))
+        cfg = RetryConfig(
+            max_retries=3,
+            circuit_breaker=name,
+            base_delay=0.01,
+            jitter=Jitter.NONE,
+        )
+        with pytest.raises(RuntimeError, match="Circuit breaker"):
+            await async_retry(fn, config=cfg)
+        assert fn.call_count == 0
+        del resilience.registry.breakers[name]
+
+    @pytest.mark.asyncio
+    async def test_circuit_breaker_records_success(self) -> None:
+        name = "cb_success_utest"
+        fn = AsyncMock(return_value="ok")
+        cfg = RetryConfig(
+            max_retries=1,
+            circuit_breaker=name,
+            base_delay=0.01,
+            jitter=Jitter.NONE,
+        )
+        result = await async_retry(fn, config=cfg)
+        assert result == "ok"
+        cb = resilience.registry.breaker(name)
+        assert cb.successes >= 1
+        del resilience.registry.breakers[name]
+
+    @pytest.mark.asyncio
+    async def test_circuit_breaker_records_failure(self) -> None:
+        name = "cb_fail_utest"
+        fn = AsyncMock(side_effect=ConnectionError("fail"))
+        cfg = RetryConfig(
+            max_retries=1,
+            circuit_breaker=name,
+            base_delay=0.01,
+            jitter=Jitter.NONE,
+        )
+        with pytest.raises(ConnectionError):
+            await async_retry(fn, config=cfg)
+        cb = resilience.registry.breaker(name)
+        assert cb.failures >= 1
+        del resilience.registry.breakers[name]
+
+    @pytest.mark.asyncio
+    async def test_default_config_works(self) -> None:
+        fn = AsyncMock(return_value="ok")
+        result = await async_retry(fn)
+        assert result == "ok"
+
+    @pytest.mark.asyncio
+    async def test_passes_args_and_kwargs(self) -> None:
+        fn = AsyncMock(return_value="ok")
+
+        async def target(url: str, *, timeout: int = 30) -> str:
+            fn(url, timeout=timeout)
+            return "ok"
+
+        await async_retry(
+            target,
+            "https://example.com",
+            config=RetryConfig(jitter=Jitter.NONE),
+            timeout=10,
+        )
+        fn.assert_called_once_with("https://example.com", timeout=10)
+
+
+class TestMakeRetryDecorator:
+    @pytest.mark.asyncio
+    async def test_decorator_retries_on_failure(self) -> None:
+        call_count = 0
+
+        @make_retry_decorator(RetryConfig(max_retries=2, base_delay=0.01, jitter=Jitter.NONE))
+        async def flaky_func() -> str:
+            nonlocal call_count
+            call_count += 1
+            if call_count < 3:
+                raise ValueError("not yet")
+            return "success"
+
+        result = await flaky_func()
+        assert result == "success"
+        assert call_count == 3
+
+    @pytest.mark.asyncio
+    async def test_decorator_preserves_metadata(self) -> None:
+        @make_retry_decorator()
+        async def my_func() -> str:
+            """My docstring."""
+            return "ok"
+
+        assert my_func.__name__ == "my_func"
+        assert my_func.__doc__ == "My docstring."

tests/test_settings.py

@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+"""Tests for unified Pry settings."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from settings import _NESTED_KEY_MAP, PrySettings
+
+
+@pytest.fixture
+def isolated_settings(tmp_path: Path) -> PrySettings:
+    """Return a fresh settings instance with a temp config file."""
+    config_file = tmp_path / "config.json"
+    return PrySettings(config_file=config_file)
+
+
+def test_settings_get_default(isolated_settings: PrySettings) -> None:
+    assert isolated_settings.max_retries == 3
+    assert isolated_settings.rate_limit_rpm == 120
+
+
+def test_settings_proxy_resolution(isolated_settings: PrySettings) -> None:
+    assert isolated_settings.resolved_proxy_url is None
+    isolated_settings.proxy_enabled = True
+    isolated_settings.proxy_url = "http://proxy.example:8080"
+    assert isolated_settings.resolved_proxy_url == "http://proxy.example:8080"
+
+
+def test_settings_tor_chain(isolated_settings: PrySettings) -> None:
+    isolated_settings.tor_enabled = True
+    isolated_settings.proxy_enabled = True
+    isolated_settings.proxy_url = "http://proxy.example:8080"
+    chain = isolated_settings.resolved_proxy_chain
+    assert len(chain) == 2
+    assert chain[0].startswith("socks5://")
+
+
+def test_settings_to_api_dict_excludes_private(isolated_settings: PrySettings) -> None:
+    data = isolated_settings.to_api_dict()
+    assert "_proxy_chain" not in data
+    assert "proxy" in data
+    assert isinstance(data["proxy"]["enabled"], bool)
+
+
+def test_settings_update_and_persist(isolated_settings: PrySettings) -> None:
+    result = isolated_settings.update({"retry": {"max_attempts": 5}})
+    assert result["status"] == "ok"
+    assert isolated_settings.max_retries == 5
+    assert isolated_settings.config_file.exists()
+    saved = json.loads(isolated_settings.config_file.read_text(encoding="utf-8"))
+    assert saved["retry"]["max_attempts"] == 5
+
+
+def test_settings_load_config_file(tmp_path: Path) -> None:
+    config_file = tmp_path / "config.json"
+    config_file.write_text(
+        json.dumps({"retry": {"max_attempts": 7}, "tor": {"enabled": True}}),
+        encoding="utf-8",
+    )
+    s = PrySettings(config_file=config_file)
+    assert s.max_retries == 7
+    assert s.tor_enabled is True
+
+
+def test_settings_legacy_env_vars(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+    monkeypatch.setenv("PROXY_URL", "http://legacy-proxy:8080")
+    monkeypatch.setenv("TOR_ENABLED", "true")
+    monkeypatch.setenv("MAX_RETRIES", "9")
+    config_file = tmp_path / "config.json"
+    s = PrySettings(config_file=config_file)
+    assert s.proxy_url == "http://legacy-proxy:8080"
+    assert s.tor_enabled is True
+    assert s.max_retries == 9
+
+
+def test_settings_enable_tor(isolated_settings: PrySettings) -> None:
+    result = isolated_settings.enable_tor()
+    assert result["status"] == "ok"
+    assert isolated_settings.tor_enabled is True
+    assert isolated_settings.proxy_enabled is True
+    assert isolated_settings.proxy_url == "socks5://tor:9050"
+
+
+def test_nested_key_map_complete() -> None:
+    """Ensure every API-facing nested key maps to a settings attribute."""
+    for attr in _NESTED_KEY_MAP.values():
+        assert attr in PrySettings.model_fields, f"PrySettings missing {attr}"

tests/test_ultimate_scraper.py

@@ -1,13 +1,41 @@
+"""Tests for ultimate scraper fallback chain."""
+
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2026 Rug Munch Media LLC
 #
 # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
 # Licensed under MIT. See LICENSE.
-"""Tests for ultimate scraper."""
 
+from __future__ import annotations
+
+from contextlib import ExitStack
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+import resilience
 from ultimate_scraper import UltimateScraper
 
 
+def _clear_registry() -> None:
+    """Reset global resilience registry to avoid cross-test contamination."""
+    resilience.registry.domain_memory.data.clear()
+    for name in list(resilience.registry.breakers.keys()):
+        cb = resilience.registry.breakers[name]
+        cb.state = "closed"
+        cb.failures = 0
+        cb.successes = 0
+        cb.last_failure_at = 0.0
+        cb.half_open_attempts = 0
+
+
+_TIER_ALL = [
+    "_tier_direct", "_tier_tls_fingerprint", "_tier_cloudscraper",
+    "_tier_flaresolverr", "_tier_undetected", "_tier_camoufox",
+    "_tier_playwright", "_tier_googlebot", "_tier_premium_proxy",
+    "_tier_archive", "_tier_google_cache", "_tier_tor",
+]
+
 def test_ultimate_init() -> None:
     s = UltimateScraper()
     assert s is not None
@@ -27,3 +55,123 @@ def test_is_valid_empty() -> None:
     assert s._is_valid("") is False
     assert s._is_valid("<html></html>") is False
     assert s._is_valid(None) is False
+
+
+@pytest.mark.anyio
+async def test_tier_tls_fingerprint_not_available() -> None:
+    s = UltimateScraper()
+    s._tls_scraper = None
+    result = await s._tier_tls_fingerprint("http://example.com", {}, None)
+    assert result is None
+
+
+@pytest.mark.anyio
+async def test_tier_tls_fingerprint_success() -> None:
+    s = UltimateScraper()
+    mock = MagicMock()
+    mock.is_available.return_value = True
+    mock.fetch = AsyncMock(
+        return_value={
+            "success": True,
+            "status_code": 200,
+            "text": "<html><body>hello world</body></html>",
+        }
+    )
+    s._tls_scraper = mock
+    result = await s._tier_tls_fingerprint("http://example.com", {}, None)
+    assert result == "<html><body>hello world</body></html>"
+
+
+@pytest.mark.anyio
+async def test_tier_tls_fingerprint_failure() -> None:
+    s = UltimateScraper()
+    mock = MagicMock()
+    mock.is_available.return_value = True
+    mock.fetch = AsyncMock(return_value={"success": False, "error": "blocked"})
+    s._tls_scraper = mock
+    result = await s._tier_tls_fingerprint("http://example.com", {}, None)
+    assert result is None
+
+
+@pytest.mark.anyio
+async def test_tier_camoufox_not_available() -> None:
+    s = UltimateScraper()
+    s._camoufox = None
+    result = await s._tier_camoufox("http://example.com", {}, None)
+    assert result is None
+
+
+@pytest.mark.anyio
+async def test_tier_camoufox_success() -> None:
+    s = UltimateScraper()
+    mock = MagicMock()
+    mock.is_available.return_value = True
+    mock.fetch = AsyncMock(
+        return_value={
+            "success": True,
+            "content": "<html><body>camoufox result</body></html>",
+        }
+    )
+    s._camoufox = mock
+    result = await s._tier_camoufox("http://example.com", {}, None)
+    assert result == "<html><body>camoufox result</body></html>"
+
+
+@pytest.mark.anyio
+async def test_tier_camoufox_failure() -> None:
+    s = UltimateScraper()
+    mock = MagicMock()
+    mock.is_available.return_value = True
+    mock.fetch = AsyncMock(return_value={"success": False, "error": "blocked"})
+    s._camoufox = mock
+    result = await s._tier_camoufox("http://example.com", {}, None)
+    assert result is None
+
+
+@pytest.mark.anyio
+async def test_scrape_falls_through_all_tiers() -> None:
+    _clear_registry()
+    s = UltimateScraper()
+    with patch.object(s, "_tier_direct", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_tls_fingerprint", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_cloudscraper", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_flaresolverr", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_undetected", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_camoufox", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_playwright", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_googlebot", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_archive", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_google_cache", new=AsyncMock(return_value=None)), \
+         patch.object(s, "_tier_tor", new=AsyncMock(return_value=None)):
+        result = await s.scrape("http://example.com")
+        assert result["status"] == "error"
+
+
+@pytest.mark.anyio
+async def test_scrape_tls_tier_succeeds() -> None:
+    _clear_registry()
+    s = UltimateScraper()
+    html = "<html><body>" + "x" * 1000 + "</body></html>"
+    ctx_mocks = {name: AsyncMock(return_value=None) for name in _TIER_ALL}
+    ctx_mocks["_tier_tls_fingerprint"].return_value = html
+    with ExitStack() as stack:
+        for name in _TIER_ALL:
+            stack.enter_context(patch.object(s, name, new=ctx_mocks[name]))
+        result = await s.scrape("http://example.com")
+        assert result["status"] == "ok"
+        assert result["method"] == "tls_fingerprint"
+
+
+@pytest.mark.anyio
+async def test_scrape_camoufox_tier_succeeds() -> None:
+    _clear_registry()
+    s = UltimateScraper()
+    html = "<html><body>" + "x" * 1000 + "</body></html>"
+    ctx_mocks = {name: AsyncMock(return_value=None) for name in _TIER_ALL}
+    ctx_mocks["_tier_camoufox"].return_value = html
+    with ExitStack() as stack:
+        for name in _TIER_ALL:
+            stack.enter_context(patch.object(s, name, new=ctx_mocks[name]))
+        result = await s.scrape("http://example.com")
+        assert result["status"] == "ok"
+        assert result["method"] == "camoufox"

tests/test_url_guard.py

@@ -0,0 +1,107 @@
+"""Tests for SSRF and path-traversal guard."""
+
+from __future__ import annotations
+
+import pytest
+
+from errors import InvalidRequestError
+from url_guard import validate_url
+
+
+class TestValidateURL:
+    def test_accepts_normal_https(self):
+        result = validate_url("https://example.com/page")
+        assert result == "https://example.com/page"
+
+    def test_accepts_normal_http(self):
+        result = validate_url("http://example.com/page")
+        assert result == "http://example.com/page"
+
+    def test_accepts_https_with_path_and_query(self):
+        url = "https://api.example.com/v1/data?key=val&page=2#section"
+        result = validate_url(url)
+        assert result == url
+
+    def test_rejects_empty_url(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("")
+
+    def test_rejects_ftp_scheme(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("ftp://example.com/file")
+
+    def test_rejects_file_scheme(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("file:///etc/passwd")
+
+    def test_rejects_javascript_scheme(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("javascript:alert(1)")
+
+    def test_rejects_localhost_hostname(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://localhost:8080/admin")
+
+    def test_rejects_localhost_ip(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://127.0.0.1:8080/admin")
+
+    def test_rejects_private_ip_10_dot(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://10.0.0.1/admin")
+
+    def test_rejects_private_ip_192_168(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://192.168.1.1/admin")
+
+    def test_rejects_private_ip_172(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://172.16.0.1/admin")
+
+    def test_rejects_internal_hostname(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://internal.corp/admin")
+
+    def test_rejects_kubernetes_metadata(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://kubernetes.default.svc.cluster.local")
+
+    def test_rejects_embedded_credentials(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://user:pass@example.com/")
+
+    def test_rejects_path_traversal_simple(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("https://example.com/../../etc/passwd")
+
+    def test_rejects_path_traversal_encoded(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("https://example.com/%2e%2e%2fetc/passwd")
+
+    def test_rejects_path_traversal_double_encoded(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("https://example.com/%252e%252e%252fetc")
+
+    def test_rejects_tailscale_ip(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://100.64.0.1/")
+
+    def test_rejects_empty_hostname(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http:///path")
+
+    def test_rejects_docker_host(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://docker/container")
+
+    def test_rejects_metadata_google(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://metadata.google.internal/")
+
+    def test_rejects_link_local_ipv6(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://[fe80::1]/")
+
+    def test_rejects_loopback_ipv6(self):
+        with pytest.raises(InvalidRequestError):
+            validate_url("http://[::1]/")

tests/test_x402_mcp_spec.py

@@ -10,7 +10,7 @@ import base64
 import json
 from typing import Any
 
-from mcp_production import (
+from pry_mcp import (
     MCP_PROTOCOL_VERSION,
     PRY_PROMPTS,
     PRY_RESOURCES,

ultimate_scraper.py

@@ -14,9 +14,16 @@ import logging
 import os
 import random
 import re
+import time
 from typing import Any, ClassVar
 from urllib.parse import urlparse
 
+from errors import InvalidRequestError
+from observability import SCRAPE_LATENCY
+from resilience import registry
+from retry import RetryConfig, async_retry
+from url_guard import validate_url
+
 logger = logging.getLogger(__name__)
 
 # Try importing optional bypass libraries
@@ -28,21 +35,44 @@ _has_tor = importlib.util.find_spec("aiohttp_socks") is not None
 if _has_playwright:
     from playwright.async_api import async_playwright
 
+try:
+    from tls_fingerprint import TLSScraper
+
+    _has_tls = True
+except ImportError:
+    _has_tls = False
+
+try:
+    from camoufox_integration import CamoufoxBrowser
+
+    _has_camoufox = True
+except ImportError:
+    _has_camoufox = False
+
+try:
+    from stealth_engine import StealthEngine
+
+    _stealth_engine = StealthEngine()
+except ImportError:
+    _stealth_engine = None
+
 
 class UltimateScraper:
-    """10-tier anti-detection scraper with automatic fallback.
+    """12-tier anti-detection scraper with automatic fallback.
 
     Tiers:
     1. Direct HTTP (smart headers, rotating UAs)
-    2. cloudscraper (Python-native Cloudflare bypass)
-    3. FlareSolverr (Cloudflare/WAF bypass service)
-    4. undetected-chromedriver (modified Chrome, no detection)
-    5. Playwright stealth (full browser with human behavior)
-    6. Googlebot UA (search engine crawl mimic)
-    7. Tor proxy (anonymous routing via SOCKS5)
-    8. Archive.org / Wayback Machine (cached version)
-    9. Google Cache (cached version)
-    10. Textise dot iitty (text-only version)
+    2. TLS fingerprint impersonation (curl_cffi JA3/JA4 bypass)
+    3. cloudscraper (Python-native Cloudflare bypass)
+    4. FlareSolverr (Cloudflare/WAF bypass service)
+    5. undetected-chromedriver (modified Chrome, no detection)
+    6. Camoufox stealth Firefox (source-level anti-fingerprinting)
+    7. Playwright stealth (full browser with human behavior)
+    8. Googlebot UA (search engine crawl mimic)
+    9. Premium proxy retry (after cheap tiers fail)
+    10. Archive.org / Wayback Machine (cached version)
+    11. Google Cache (cached version)
+    12. Tor proxy (anonymous routing via SOCKS5)
     """
 
     USER_AGENTS: ClassVar[list[str]] = [
@@ -54,6 +84,27 @@ class UltimateScraper:
 
     def __init__(self) -> None:
         self._ua_index = 0
+        self._tls_scraper = TLSScraper() if _has_tls else None
+        self._camoufox = CamoufoxBrowser() if _has_camoufox else None
+
+    # Per-tier retry configurations for transient failure resilience.
+    # Browser-based tiers (undetected, camoufox, playwright) use 0 retries
+    # because launching a browser is expensive; network-only tiers retry 1-2x.
+    _RETRY_CONFIGS: ClassVar[dict[str, RetryConfig]] = {
+        "direct": RetryConfig(max_retries=2, base_delay=0.5, max_delay=5.0),
+        "tls_fingerprint": RetryConfig(max_retries=1, base_delay=1.0, max_delay=5.0),
+        "cloudscraper": RetryConfig(max_retries=1, base_delay=1.0, max_delay=5.0),
+        "flaresolverr": RetryConfig(max_retries=2, base_delay=2.0, max_delay=10.0),
+        "undetected": RetryConfig(max_retries=0),
+        "camoufox": RetryConfig(max_retries=0),
+        "playwright": RetryConfig(max_retries=0),
+        "googlebot": RetryConfig(max_retries=2, base_delay=0.5, max_delay=5.0),
+        "premium_proxy": RetryConfig(max_retries=2, base_delay=1.0, max_delay=8.0),
+        "archive": RetryConfig(max_retries=2, base_delay=0.5, max_delay=5.0),
+        "google_cache": RetryConfig(max_retries=2, base_delay=0.5, max_delay=5.0),
+        "tor": RetryConfig(max_retries=1, base_delay=2.0, max_delay=10.0),
+    }
+    _DEFAULT_RETRY: ClassVar[RetryConfig] = RetryConfig(max_retries=1, base_delay=1.0)
 
     def _rotate_ua(self) -> str:
         ua = self.USER_AGENTS[self._ua_index % len(self.USER_AGENTS)]
@@ -83,6 +134,10 @@ class UltimateScraper:
         Will try up to 10 strategies until one succeeds."""
         opts = options or {}
         errors: list[str] = []
+        try:
+            validate_url(url)
+        except InvalidRequestError as e:
+            return {"status": "error", "url": url, "error": str(e)}
 
         proxy_url = self._resolve_proxy_url(opts)
 
@@ -93,57 +148,80 @@ class UltimateScraper:
                 return self._result(url, html, "premium_proxy")
             errors.append("premium_proxy_failed")
 
-        # Tier 1: Direct HTTP
-        html = await self._tier_direct(url, opts, proxy_url=proxy_url)
-        if html and self._is_valid(html):
-            return self._result(url, html, "direct")
+        # Build tier list. Order starts cheap/direct and escalates.
+        tier_order = [
+            ("direct", self._tier_direct),
+            ("tls_fingerprint", self._tier_tls_fingerprint),
+            ("cloudscraper", self._tier_cloudscraper),
+            ("flaresolverr", self._tier_flaresolverr),
+            ("undetected", self._tier_undetected),
+            ("camoufox", self._tier_camoufox),
+            ("playwright", self._tier_playwright),
+            ("googlebot", self._tier_googlebot),
+        ]
 
-        # Tier 2: cloudscraper (Python Cloudflare bypass)
-        html = await self._tier_cloudscraper(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "cloudscraper")
-
-        # Tier 3: FlareSolverr
-        html = await self._tier_flaresolverr(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "flaresolverr")
-
-        # Tier 4: undetected-chromedriver
-        html = await self._tier_undetected(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "undetected")
-
-        # Tier 5: Playwright stealth
-        html = await self._tier_playwright(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "playwright")
-
-        # Tier 6: Googlebot UA
-        html = await self._tier_googlebot(url, opts, proxy_url=proxy_url)
-        if html and self._is_valid(html):
-            return self._result(url, html, "googlebot")
-
-        # Tier 7: Premium proxy retry (after cheap tiers fail)
+        # Premium proxy retry (after cheap tiers fail) if not tried first.
         if proxy_url and not opts.get("use_premium_proxy_first", True):
-            html = await self._tier_premium_proxy(url, proxy_url, opts)
-            if html and self._is_valid(html):
-                return self._result(url, html, "premium_proxy")
-            errors.append("premium_proxy_failed")
+            tier_order.append(("premium_proxy", self._tier_premium_proxy))
 
-        # Tier 8: Archive.org / Wayback Machine
-        html = await self._tier_archive(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "archive")
+        tier_order.extend(
+            [
+                ("archive", self._tier_archive),
+                ("google_cache", self._tier_google_cache),
+                ("tor", self._tier_tor),
+            ]
+        )
 
-        # Tier 9: Google Cache
-        html = await self._tier_google_cache(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "google_cache")
+        # If domain memory knows a good tier, try it first.
+        parsed = urlparse(url)
+        domain = parsed.netloc
+        best = registry.best_tier_for_domain(domain, [name for name, _ in tier_order])
+        if best:
+            tier_order.sort(key=lambda item: (item[0] != best, item[0]))
 
-        # Tier 10: Tor (anonymous routing)
-        html = await self._tier_tor(url, opts)
-        if html and self._is_valid(html):
-            return self._result(url, html, "tor")
+        # External service tiers protected by circuit breakers.
+        breaker_for_tier = {
+            "flaresolverr": "flaresolverr",
+            "playwright": "playwright",
+            "camoufox": "camoufox",
+            "tls_fingerprint": "tls_fingerprint",
+        }
+
+        for tier_name, tier_fn in tier_order:
+            # Skip external tiers whose circuit breaker is open.
+            breaker_name = breaker_for_tier.get(tier_name)
+            if breaker_name and not registry.is_service_allowed(breaker_name):
+                logger.debug(
+                    "tier_skipped_circuit_open", extra={"tier": tier_name, "domain": domain}
+                )
+                errors.append(f"{tier_name}: circuit_open")
+                continue
+
+            start = time.time()
+            try:
+                retry_cfg = self._RETRY_CONFIGS.get(tier_name, self._DEFAULT_RETRY)
+                if tier_name == "premium_proxy":
+                    html = await async_retry(tier_fn, url, proxy_url, opts, config=retry_cfg)
+                else:
+                    html = await async_retry(
+                        tier_fn, url, opts, proxy_url=proxy_url, config=retry_cfg
+                    )
+            except Exception as e:  # noqa: BLE001
+                logger.debug("tier_exception", extra={"tier": tier_name, "error": str(e)[:50]})
+                html = None
+
+            latency = time.time() - start
+            success = bool(html and self._is_valid(html))
+            registry.record_domain_tier_result(domain, tier_name, success, latency)
+            if breaker_name:
+                registry.record_service_result(breaker_name, success)
+            if SCRAPE_LATENCY:
+                SCRAPE_LATENCY.labels(method=tier_name).observe(latency)
+
+            if success:
+                method = "premium_proxy" if tier_name == "premium_proxy" else tier_name
+                return self._result(url, html, method)
+            errors.append(f"{tier_name}_failed")
 
         last_err = errors[-1] if errors else "all_tiers_failed"
         return {
@@ -266,7 +344,9 @@ class UltimateScraper:
             logger.debug("premium_proxy_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_cloudscraper(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_cloudscraper(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         if not _has_cloudscraper:
             return None
         try:
@@ -282,7 +362,9 @@ class UltimateScraper:
             logger.debug("cloudscraper_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_flaresolverr(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_flaresolverr(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         from settings import settings
 
         if not settings.flaresolverr_url:
@@ -302,7 +384,9 @@ class UltimateScraper:
             logger.debug("flaresolverr_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_undetected(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_undetected(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         if not _has_undetected:
             return None
         try:
@@ -329,7 +413,9 @@ class UltimateScraper:
             logger.debug("undetected_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_playwright(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_playwright(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         if not _has_playwright:
             return None
         try:
@@ -352,10 +438,9 @@ class UltimateScraper:
                     locale="en-US",
                     timezone_id="America/New_York",
                 )
-                await context.add_init_script("""
-                    Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
-                    navigator.permissions.query = (p) => p.name === 'notifications' ? Promise.resolve({state: 'denied'}) : navigator.permissions.query(p);
-                """)
+                stealth_script = _stealth_engine.generate_all() if _stealth_engine else ""
+                if stealth_script:
+                    await context.add_init_script(stealth_script)
                 page = await context.new_page()
                 await page.goto(url, wait_until="domcontentloaded", timeout=60000)
                 await page.wait_for_timeout(random.randint(1000, 3000))
@@ -370,6 +455,43 @@ class UltimateScraper:
             logger.debug("playwright_failed", extra={"error": str(e)[:50]})
         return None
 
+    async def _tier_tls_fingerprint(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
+        """Fetch using curl_cffi TLS fingerprint impersonation."""
+        if not self._tls_scraper or not self._tls_scraper.is_available():
+            return None
+        try:
+            result = await self._tls_scraper.fetch(
+                url,
+                headers=self._build_headers(url),
+                proxy=proxy_url or "",
+                timeout=opts.get("timeout", 30),
+            )
+            if result.get("success") and result.get("status_code", 0) < 400:
+                return result.get("text", "")
+        except Exception as e:  # noqa: BLE001
+            logger.debug("tls_fingerprint_failed", extra={"error": str(e)[:50]})
+        return None
+
+    async def _tier_camoufox(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
+        """Fetch using Camoufox anti-detection Firefox."""
+        if not self._camoufox or not self._camoufox.is_available():
+            return None
+        try:
+            result = await self._camoufox.fetch(
+                url,
+                wait_time=opts.get("wait_time", 3000),
+                proxy=proxy_url or "",
+            )
+            if result.get("success"):
+                return result.get("content", "")
+        except Exception as e:  # noqa: BLE001
+            logger.debug("camoufox_failed", extra={"error": str(e)[:50]})
+        return None
+
     async def _tier_googlebot(
         self, url: str, opts: dict[str, Any], proxy_url: str | None = None
     ) -> str | None:
@@ -388,7 +510,9 @@ class UltimateScraper:
             logger.debug("googlebot_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_archive(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_archive(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         """Fetch from Wayback Machine CDX API for latest snapshot."""
         try:
             import httpx
@@ -408,7 +532,9 @@ class UltimateScraper:
             logger.debug("archive_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_google_cache(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_google_cache(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         try:
             import httpx
 
@@ -421,7 +547,9 @@ class UltimateScraper:
             logger.debug("google_cache_failed", extra={"error": str(e)[:50]})
         return None
 
-    async def _tier_tor(self, url: str, opts: dict[str, Any]) -> str | None:
+    async def _tier_tor(
+        self, url: str, opts: dict[str, Any], proxy_url: str | None = None
+    ) -> str | None:
         """Fetch via Tor SOCKS5 proxy."""
         if not _has_tor:
             return None

url_guard.py

@@ -0,0 +1,155 @@
+"""SSRF and path traversal guard for Pry scrapers.
+
+Validates URLs before scraping to prevent:
+- SSRF attacks (private IPs, loopback, internal hosts)
+- Path traversal (../, encoded variants)
+- File:// or other non-HTTP schemes
+- DNS rebinding risks (common internal hostnames)
+"""
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import re
+import socket
+from urllib.parse import urlparse
+
+logger = logging.getLogger(__name__)
+
+# Private/reserved IP ranges (RFC 1918, RFC 6598, RFC 4193, link-local, loopback)
+_PRIVATE_NETS: list[str] = [
+    "127.0.0.0/8",       # loopback
+    "10.0.0.0/8",        # RFC 1918
+    "172.16.0.0/12",     # RFC 1918
+    "192.168.0.0/16",    # RFC 1918
+    "100.64.0.0/10",     # RFC 6598 (CGNAT)
+    "169.254.0.0/16",    # link-local
+    "fc00::/7",          # RFC 4193 (ULA)
+    "fe80::/10",         # link-local IPv6
+    "::1/128",           # IPv6 loopback
+]
+
+# Common internal hostnames that suggest SSRF targeting
+_INTERNAL_HOST_RE = re.compile(
+    r"^(.+\.)?"
+    r"(localhost|internal|intranet|private|corp|home|router|gateway|"
+    r"metadata|kubernetes|kube|docker|consul|nomad|vault|"
+    r"elasticsearch|redis|mysql|postgres|memcached|"
+    r"grafana|prometheus|alertmanager|"
+    r"minio|s3|storage|db|database|"
+    r"admin|dashboard|management|"
+    r"apiserver|etcd|"
+    r"tailscale|100\.\d+\.\d+\.\d+)"
+    r"(\..+)?$",
+    re.IGNORECASE,
+)
+
+# Blocked URL schemes
+_ALLOWED_SCHEMES = {"http", "https"}
+
+# Path traversal patterns
+_TRAVERSAL_RE = re.compile(
+    r"(?:^|/)\.\.(?:/|%2[fF]|\\|$)"
+    r"|%2[eE]%2[eE]"
+    r"|%252[eE]%252[eE]%252[fF]"
+    r"|\.\.(?:/|\\|$)"
+)
+
+
+def validate_url(url: str, *, resolve_dns: bool = False) -> str:
+    """Validate a URL for SSRF and path-traversal safety.
+
+    Args:
+        url: The URL to validate.
+        resolve_dns: If True, perform a DNS lookup to check that the resolved
+                     IP is not in a private range. This adds latency and cannot
+                     be async here, so it defaults to False.
+
+    Returns:
+        The validated URL (unchanged).
+
+    Raises:
+        InvalidRequestError: If the URL is unsafe.
+    """
+    if not url:
+        raise _error("URL is empty", url)
+
+    parsed = urlparse(url)
+
+    # Reject non-HTTP schemes
+    if parsed.scheme not in _ALLOWED_SCHEMES:
+        raise _error(
+            f"URL scheme {parsed.scheme} is not allowed (only http/https)",
+            url,
+        )
+
+    # Reject URLs with embedded credentials
+    if parsed.username or parsed.password:
+        raise _error("URL must not contain embedded credentials (user:password@)", url)
+
+    hostname = parsed.hostname or ""
+
+    # Reject empty hostnames
+    if not hostname:
+        raise _error("URL must have a valid hostname", url)
+
+    # Reject raw IP: check if it is a private/reserved address
+    try:
+        ip = ipaddress.ip_address(hostname)
+        for net in _PRIVATE_NETS:
+            if ip in ipaddress.ip_network(net):
+                raise _error(
+                    f"URL resolves to a private/reserved IP range ({net})",
+                    url,
+                )
+    except ValueError:
+        # Not an IP address — continue with hostname checks
+        pass
+
+    # Reject internal/sensitive hostnames
+    if _INTERNAL_HOST_RE.match(hostname):
+        raise _error(
+            f"URL hostname {hostname} appears to be an internal resource",
+            url,
+        )
+
+    # Reject DNS resolution to private IPs (opt-in)
+    if resolve_dns:
+        try:
+            addrs = socket.getaddrinfo(hostname, None)
+            for _family, _, _, _, sockaddr in addrs:
+                addr = sockaddr[0]
+                try:
+                    ip = ipaddress.ip_address(addr)
+                    for net in _PRIVATE_NETS:
+                        if ip in ipaddress.ip_network(net):
+                            raise _error(
+                                f"URL resolves to private IP {addr} ({net})",
+                                url,
+                            )
+                except ValueError:
+                    continue
+        except OSError as e:
+            logger.debug("dns_lookup_failed", extra={"hostname": hostname, "error": str(e)})
+
+    # Check path for traversal attempts
+    path = parsed.path + parsed.params + parsed.query + parsed.fragment
+    if _TRAVERSAL_RE.search(path):
+        raise _error("URL contains path traversal sequences", url)
+
+    return url
+
+
+def _error(message: str, url: str) -> Exception:
+    """Build a structured InvalidRequestError."""
+    from errors import InvalidRequestError
+
+    logger.warning("url_guard_rejected", extra={"url": url, "reason": message})
+    return InvalidRequestError(message, details={"url": url, "reason": message})

x402_middleware.py

@@ -132,7 +132,7 @@ class X402Middleware:
                         await self.app(scope, receive, wrapped_send)
                         return
             except (ValueError, KeyError, TypeError) as e:
-                logger.warning("x402_payment_decode_failed err=%s", e)
+                logger.warning("x402_payment_decode_failed", extra={"error": str(e)})
 
         operation = next((op for ep, op in PAID_ENDPOINTS.items() if path.startswith(ep)), None)
         if not operation or operation not in X402_PRICING: