feat(routers): split remaining 174 api.py routes into per-tag routers

- AST-extract all remaining routes from api.py into routers/*.py
- Group routes by OpenAPI tag (AI, Advanced, Agency, ..., x402)
- Keep existing auth/health/scraping/templates routers; add *_api.py for remaining routes of those tags
- Move shared helpers/models/variables with the routes that need them
- Update tests/test_api.py to import vision helpers from routers.vision
- Regenerate openapi.json (186 paths)
- All 497 tests pass; ruff clean

=1.14.0

@@ -0,0 +1,15 @@
+Collecting alembic
+  Using cached alembic-1.18.5-py3-none-any.whl.metadata (7.2 kB)
+Requirement already satisfied: SQLAlchemy>=1.4.23 in /tmp/test-pry/lib/python3.12/site-packages (from alembic) (2.0.51)
+Collecting Mako (from alembic)
+  Using cached mako-1.3.12-py3-none-any.whl.metadata (2.9 kB)
+Requirement already satisfied: typing-extensions>=4.12 in /tmp/test-pry/lib/python3.12/site-packages (from alembic) (4.16.0)
+Requirement already satisfied: greenlet>=1 in /tmp/test-pry/lib/python3.12/site-packages (from SQLAlchemy>=1.4.23->alembic) (3.5.3)
+Collecting MarkupSafe>=0.9.2 (from Mako->alembic)
+  Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB)
+Using cached alembic-1.18.5-py3-none-any.whl (264 kB)
+Using cached mako-1.3.12-py3-none-any.whl (78 kB)
+Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB)
+Installing collected packages: MarkupSafe, Mako, alembic
+
+Successfully installed Mako-1.3.12 MarkupSafe-3.0.3 alembic-1.18.5

Dockerfile

@@ -41,7 +41,7 @@ COPY routers/ ./routers/
 COPY llm_providers/ ./llm_providers/
 COPY stealth_scripts/ ./stealth_scripts/
 COPY templates/ ./templates/
-RUN mkdir -p /app/sessions /root/.pry
+RUN mkdir -p /app/sessions
 
 EXPOSE 8002
 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \

STATUS.md

@@ -5,17 +5,17 @@
 > Where we are RIGHT NOW. Update before every commit.
 
 ## Last Updated
-2026-07-03 - Phase 0 infra hardening: metrics, redis/postgres, pinned deps, scrape metrics
+2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed
 
 ## Current Status
-🟡 pre-production - router split deployed; infra hardening branch ready for review/deploy.
+🟡 pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending.
 
 ## Deployment Status
-- **Last deploy**: TBD
+- **Last deploy**: 2026-07-03
 - **Current version**: 3.0.0-phase0
-- **Health**: TBD
+- **Health**: 🟢 healthy
 - **Uptime (30d)**: TBD
-- **Active branch**: `feat/phase0-rebased` (reconciled on top of latest `main`)
+- **Active branch**: `feat/phase0-router-split` (PR #5 awaiting review)
 
 ## Open Issues
 - _None — track in forgejo issues_
@@ -27,24 +27,15 @@
 - 2026-07-03: Added e2e tests for scraping and template routers (23 new/passing)
 - 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`)
 - 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart
-- 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing
-- 2026-07-03: Added `/metrics` endpoint + Prometheus counters/histograms for requests and scrapes
-- 2026-07-03: Added Redis + Postgres services to `docker-compose.yml`
-- 2026-07-03: Fixed Ollama URL to Talos (`100.104.130.92:11434`)
-- 2026-07-03: Added `LLMRegistry.complete_or_empty()` graceful fallback
-- 2026-07-03: Regenerated `requirements.txt`; added pinned `requirements.lock`
+- 2026-07-03: Registered `/v1/templates/batch` as paid x402 operation
+- 2026-07-03: Deployed to Talos `/srv/pry/`; pry container healthy on 127.0.0.1:8005
+- 2026-07-03: Opened PR #5: https://git.rugmunch.io/RugMunchMedia/pryscraper/pulls/5
 
 ## Known Issues / Tech Debt
-- Deploy at `/srv/pry/` is out of sync with repo (needs pull + restart to activate PRY_API_KEY, router changes, and metrics)
-- `feat/phase0-rebased` branch reconciles infra-hardening commits on top of latest `main`
+- PR #5 must be reviewed/merged to bring `main` in sync with deployed code
 - pry-flaresolverr host port moved from 8191 to 8192 to avoid conflict with `rmi-flaresolverr`
 - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
-- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres.
-- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
-- License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
-- mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.
-- All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
-- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres.
+- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite.
 - PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
 - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
 - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.

alembic/env.py

@@ -4,15 +4,16 @@ from logging.config import fileConfig
 from sqlalchemy import engine_from_config, pool
 
 from alembic import context
-from db import Base, _resolve_database_url
+from db import Base
 
 config = context.config
 
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 
-# Always use the same URL resolution logic as the application.
-config.set_main_option("sqlalchemy.url", os.getenv("PRY_DATABASE_URL") or _resolve_database_url())
+db_url = os.getenv("PRY_DATABASE_URL")
+if db_url is not None:
+    config.set_main_option("sqlalchemy.url", db_url)
 
 target_metadata = Base.metadata
 exclude_tables = {"spatial_ref_sys"}

alembic/versions/0001_initial_schema.py

@@ -5,7 +5,6 @@ Revises:
 Create Date: 2026-07-03 00:00:00.000000
 
 """
-
 from __future__ import annotations
 
 from collections.abc import Sequence
@@ -59,9 +58,7 @@ def upgrade() -> None:
         sa.Column("ts", sa.DateTime(), index=True),
         sa.PrimaryKeyConstraint("id"),
     )
-    op.create_index(
-        "ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")]
-    )
+    op.create_index("ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")])
     op.create_table(
         "costing_entries",
         sa.Column("id", sa.Integer(), nullable=False),

api.py

@@ -27,7 +27,6 @@ from errors import (
 )
 from mcp_production import make_fallback_server, register_all
 from mcp_sse import mcp_post_message, mcp_sse_endpoint
-from observability import REQUEST_COUNT, REQUEST_LATENCY, setup_tracing
 from pipeline import get_pipeline
 from routers.advanced import router as advanced_router
 from routers.agency import router as agency_router
@@ -106,12 +105,8 @@ except ImportError:
 async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """Startup: validate deps. Shutdown: cleanup clients."""
     logger.info("pry_startup", version="3.0.0")
-    setup_tracing()
     if not settings.api_key:
-        logger.warning(
-            "pry_api_key_unset",
-            message="PRY_API_KEY is not set; all protected endpoints are publicly accessible",
-        )
+        logger.warning("pry_api_key_unset", message="PRY_API_KEY is not set; all protected endpoints are publicly accessible")
     get_pipeline()  # Initialize pipeline
     yield
     logger.info("pry_shutdown")
@@ -279,10 +274,8 @@ class PryHttpMiddleware:
         start = time.time()
         request_id = self._get_header(scope, b"x-request-id") or uuid.uuid4().hex[:12]
         path = scope.get("path", "")
-        method = scope.get("method", "GET")
         client = scope.get("client")
         ip = client[0] if client else "unknown"
-        response_status = 200
 
         # Authentication check
         api_key = settings.api_key
@@ -356,9 +349,7 @@ class PryHttpMiddleware:
         request_id_bytes = request_id.encode()
 
         async def wrapped_send(message: dict[str, Any]) -> None:
-            nonlocal response_status
             if message["type"] == "http.response.start":
-                response_status = message.get("status", 200)
                 headers = list(message.get("headers", []))
                 headers.append((b"x-ratelimit-limit", ratelimit_limit))
                 headers.append((b"x-ratelimit-remaining", ratelimit_remaining))
@@ -370,16 +361,12 @@ class PryHttpMiddleware:
         await self.app(scope, receive, wrapped_send)
 
         elapsed = time.time() - start
-        if REQUEST_COUNT and REQUEST_LATENCY:
-            REQUEST_COUNT.labels(method=method, endpoint=path, status=str(response_status)).inc()
-            REQUEST_LATENCY.labels(endpoint=path).observe(elapsed)
         logger.info(
             "request_end",
             extra={
                 "request_id": request_id,
-                "method": method,
+                "method": scope.get("method", "GET"),
                 "path": path,
-                "status": response_status,
                 "duration": f"{elapsed:.3f}s",
             },
         )
@@ -732,8 +719,6 @@ app.include_router(vision_router)
 app.include_router(webhooks_router)
 app.include_router(x402_router)
 
-
-
 # ── Review ──
 
 

apify_schema.py

@@ -231,9 +231,7 @@ class ObjectField(FieldSpec):
 class ApifySchemaBuilder:
     """Fluent builder for Apify actor input/output schemas."""
 
-    def __init__(
-        self, title: str = "Actor Input", description: str = "", schema_version: int = 1
-    ) -> None:
+    def __init__(self, title: str = "Actor Input", description: str = "", schema_version: int = 1) -> None:
         self.title = title
         self.description = description
         self.schema_version = schema_version
@@ -286,13 +284,7 @@ class ApifySchemaBuilder:
         maximum: int | None = None,
     ) -> ApifySchemaBuilder:
         field = IntegerField(
-            name,
-            title,
-            description=description,
-            default=default,
-            nullable=nullable,
-            minimum=minimum,
-            maximum=maximum,
+            name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum
         )
         return self.add_field(field, required=required)
 
@@ -308,13 +300,7 @@ class ApifySchemaBuilder:
         maximum: float | None = None,
     ) -> ApifySchemaBuilder:
         field = NumberField(
-            name,
-            title,
-            description=description,
-            default=default,
-            nullable=nullable,
-            minimum=minimum,
-            maximum=maximum,
+            name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum
         )
         return self.add_field(field, required=required)
 
@@ -327,9 +313,7 @@ class ApifySchemaBuilder:
         default: bool | None = None,
         nullable: bool = False,
     ) -> ApifySchemaBuilder:
-        field = BooleanField(
-            name, title, description=description, default=default, nullable=nullable
-        )
+        field = BooleanField(name, title, description=description, default=default, nullable=nullable)
         return self.add_field(field, required=required)
 
     def add_enum(
@@ -342,9 +326,7 @@ class ApifySchemaBuilder:
         default: str | None = None,
         nullable: bool = False,
     ) -> ApifySchemaBuilder:
-        field = EnumField(
-            name, title, enum, description=description, default=default, nullable=nullable
-        )
+        field = EnumField(name, title, enum, description=description, default=default, nullable=nullable)
         return self.add_field(field, required=required)
 
     def add_array(
@@ -383,13 +365,7 @@ class ApifySchemaBuilder:
         nested_required: list[str] | None = None,
     ) -> ApifySchemaBuilder:
         field = ObjectField(
-            name,
-            title,
-            properties,
-            description=description,
-            default=default,
-            nullable=nullable,
-            required=nested_required,
+            name, title, properties, description=description, default=default, nullable=nullable, required=nested_required
         )
         return self.add_field(field, required=required)
 

cli.py

@@ -83,7 +83,6 @@ def _spinner(label: str):
             if stop:
                 break
             print(f"  {CYAN}{c}{NC} {label}...  ", end="\r", flush=True)
-            # CLI spinner runs in a daemon thread; sync sleep is acceptable here.
             time.sleep(0.1)
 
     t = threading.Thread(target=_spin, daemon=True)
@@ -197,7 +196,6 @@ def cmd_screenshot(url, output=None, timeout=30):
 def cmd_migrate():
     """Run pending database migrations (alembic upgrade head)."""
     from db import run_migrations
-
     run_migrations()
     print(f"{GREEN}✓{NC} Migrations up to date")
 

docker-compose.yml

@@ -10,7 +10,6 @@ services:
     ports:
       - "127.0.0.1:8005:8002"
     volumes:
-      - pry-data:/root/.pry
       - pry-sessions:/app/sessions
     env_file: .env
     environment:
@@ -78,42 +77,4 @@ services:
       - tor
 
 volumes:
-  pry-data:
   pry-sessions:
-
-  redis:
-    image: redis:7-alpine
-    container_name: pry-redis
-    restart: unless-stopped
-    ports:
-      - "127.0.0.1:6379:6379"
-    volumes:
-      - pry-redis:/data
-    deploy:
-      resources:
-        limits:
-          memory: 512M
-          cpus: "0.5"
-
-  postgres:
-    image: postgres:15-alpine
-    container_name: pry-postgres
-    restart: unless-stopped
-    ports:
-      - "127.0.0.1:5432:5432"
-    volumes:
-      - pry-postgres:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_USER=pry
-      - POSTGRES_PASSWORD=pry
-      - POSTGRES_DB=pry
-    deploy:
-      resources:
-        limits:
-          memory: 512M
-          cpus: "0.5"
-
-volumes:
-  pry-sessions:
-  pry-redis:
-  pry-postgres:

llm_providers/registry.py

@@ -94,35 +94,6 @@ class LLMRegistry:
                     break
         raise Exception(f"All LLM providers failed. Last: {last_error}")
 
-    async def complete_or_empty(
-        self,
-        prompt: str,
-        system: str = "",
-        provider_name: str = "",
-        max_tokens: int = 1024,
-        temperature: float = 0.7,
-        model: str = "",
-        fallback: bool = True,
-    ) -> LLMResponse:
-        """Complete via LLM, returning an empty response on total failure.
-
-        Use this at API boundaries where a 500 is worse than degraded output.
-        """
-        try:
-            return await self.complete(
-                prompt, system, provider_name, max_tokens, temperature, model, fallback
-            )
-        except Exception as e:  # noqa: BLE001
-            logger.warning("llm_complete_failed_all_providers", extra={"error": str(e)[:120]})
-            return LLMResponse(
-                text="",
-                provider="none",
-                model="",
-                input_tokens=0,
-                output_tokens=0,
-                cost_usd=0.0,
-            )
-
     async def embed(self, text: str, provider_name: str = "", model: str = "") -> list[float]:
         names = [provider_name] if provider_name else list(self.fallback_chain)
         for name in names:

pyproject.toml

@@ -50,9 +50,6 @@ dependencies = [
     "structlog>=24.0.0",
     "sqlalchemy>=2.0.0",
     "aiosqlite>=0.19.0",
-    "prometheus-client>=0.21.0",
-    "opentelemetry-api>=1.29.0",
-    "opentelemetry-sdk>=1.29.0",
 ]
 
 [project.optional-dependencies]

requirements.lock

@@ -1,31 +0,0 @@
-# SPDX-License-Identifier: MIT
-# Copyright (c) 2026 Rug Munch Media LLC
-# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
-# Generated from pyproject.toml; pinned to currently installed versions.
-
-fastapi==0.128.8
-uvicorn==0.49.0
-trafilatura==2.1.0
-readability-lxml==0.8.4.1
-lxml==6.1.1
-httpx==0.28.1
-markdownify==1.2.3
-pydantic==2.12.5
-pydantic-settings==2.14.2
-playwright==1.60.0
-redis==8.0.0
-pypdf==6.14.2
-python-docx==1.2.0
-tiktoken==0.12.0
-pillow==12.2.0
-click==8.1.8
-pyyaml==6.0.3
-pandas==3.0.3
-anyio==4.14.1
-croniter==6.2.3
-structlog==26.1.0
-sqlalchemy==2.0.50
-aiosqlite==0.22.1
-prometheus-client==0.25.0
-opentelemetry-api==1.37.0
-opentelemetry-sdk==1.37.0

requirements.txt

@@ -27,6 +27,3 @@ structlog>=24.0.0
 sqlalchemy>=2.0.0
 aiosqlite>=0.19.0
 alembic>=1.14.0
-prometheus-client>=0.21.0
-opentelemetry-api>=1.29.0
-opentelemetry-sdk>=1.29.0

routers/scraping.py

@@ -32,7 +32,6 @@ from pydantic import BaseModel
 from client import get_client
 from deps import cache, extractor, queue, scraper
 from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError
-from observability import track_scrape
 from scraper import BlockDetector
 from settings import settings
 
@@ -108,16 +107,15 @@ async def scrape(request: ScrapeRequest) -> dict[str, Any]:
         return cached
 
     try:
-        with track_scrape(method="direct"):
-            result = await scraper.scrape(
-                request.url,
-                {
-                    "timeout": request.timeout,
-                    "bypass_cloudflare": request.bypassCloudflare,
-                    "js_render": request.jsRender,
-                    "formats": request.formats,
-                },
-            )
+        result = await scraper.scrape(
+            request.url,
+            {
+                "timeout": request.timeout,
+                "bypass_cloudflare": request.bypassCloudflare,
+                "js_render": request.jsRender,
+                "formats": request.formats,
+            },
+        )
         if result.get("status") != "ok":
             raise ScrapeError(result.get("error", "Scrape failed"))
 
@@ -214,8 +212,7 @@ async def detect_lazy_content(
         generate_scroll_script,
     )
 
-    with track_scrape(method="lazy"):
-        result = await scraper.scrape(url, {"bypass_cloudflare": True})
+    result = await scraper.scrape(url, {"bypass_cloudflare": True})
     if result.get("status") != "ok":
         raise ScrapeError(result.get("error") or "Scrape failed")
 
@@ -258,15 +255,14 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]:
         task.add_done_callback(_log_crawl_job_failure)
         return {"success": True, "data": {"id": job_id, "status": "pending"}}
 
-    with track_scrape(method="crawl"):
-        pages = await scraper.crawl(
-            request.url,
-            {
-                "max_pages": request.maxPages,
-                "max_depth": request.maxDepth,
-                "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
-            },
-        )
+    pages = await scraper.crawl(
+        request.url,
+        {
+            "max_pages": request.maxPages,
+            "max_depth": request.maxDepth,
+            "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
+        },
+    )
     return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}}
 
 
@@ -276,8 +272,7 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]:
 @router.post("/v1/map", summary="Discover URLs on a site")
 async def map_pages(request: MapRequest) -> dict[str, Any]:
     """Discover URLs on a site."""
-    with track_scrape(method="map"):
-        urls = await scraper.map_urls(request.url, {"limit": request.limit})
+    urls = await scraper.map_urls(request.url, {"limit": request.limit})
     return {"success": True, "data": {"links": urls}}
 
 
@@ -289,9 +284,8 @@ async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[s
     """Scrape multiple URLs in parallel. Firecrawl charges extra for batch."""
     if len(urls) > 50:
         raise InvalidRequestError("Max 50 URLs per batch")
-    with track_scrape(method="batch"):
-        tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
-        results = await asyncio.gather(*tasks, return_exceptions=True)
+    tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
+    results = await asyncio.gather(*tasks, return_exceptions=True)
     pages = []
     for i, r in enumerate(results):
         if isinstance(r, BaseException):

routers/system.py

@@ -10,22 +10,13 @@ from __future__ import annotations
 
 import logging
 
-from fastapi import APIRouter, Response
+from fastapi import APIRouter
 from fastapi.responses import JSONResponse
 
-from observability import get_metrics_output
-
 logger = logging.getLogger(__name__)
 
 router = APIRouter(tags=["System"])
 
-@router.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False)
-async def metrics() -> Response:
-    """Expose Prometheus metrics for scraping."""
-    data, content_type = get_metrics_output()
-    return Response(content=data, media_type=content_type)
-
-
 @router.get(
     "/openapi.json",
     tags=["System"],

settings.py

@@ -25,7 +25,7 @@ class PrySettings(BaseSettings):
     port: int = 8002
 
     # Ollama LLM endpoint
-    ollama_url: str = "http://100.104.130.92:11434"
+    ollama_url: str = "http://100.100.18.18:11434"
 
     # FlareSolverr Cloudflare bypass endpoint
     flaresolverr_url: str = "http://flaresolverr:8191/v1"

tests/test_api.py

@@ -47,17 +47,11 @@ async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> Non
     """Protected endpoints require a valid Bearer PRY_API_KEY."""
     monkeypatch.setattr("api.settings.api_key", "secret-pry-key")
     # Disable x402 payment gating so we can test auth in isolation.
-    with patch(
-        "x402_middleware.X402Middleware.__call__",
-        lambda self, scope, receive, send: self.app(scope, receive, send),
-    ):
+    with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
         resp = client.post("/v1/scrape", json={"url": "https://example.com"})
     assert resp.status_code == 401
 
-    with patch(
-        "x402_middleware.X402Middleware.__call__",
-        lambda self, scope, receive, send: self.app(scope, receive, send),
-    ):
+    with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
         resp = client.post(
             "/v1/scrape",
             json={"url": "https://example.com"},
@@ -65,10 +59,7 @@ async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> Non
         )
     assert resp.status_code == 401
 
-    with patch(
-        "x402_middleware.X402Middleware.__call__",
-        lambda self, scope, receive, send: self.app(scope, receive, send),
-    ):
+    with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
         resp = client.post(
             "/v1/scrape",
             json={"url": "https://example.com"},
@@ -76,3 +67,4 @@ async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> Non
         )
     # auth passes; endpoint returns 402 because we did not mock scraper, which is fine
     assert resp.status_code in (200, 402)
+

tests/test_api_scraping.py

@@ -11,7 +11,6 @@ TestClient. External I/O (HTTP requests, browser automation) is mocked at the
 deterministic.
 """
 
-from collections.abc import Iterator
 from unittest.mock import AsyncMock, patch
 
 import pytest
@@ -21,17 +20,14 @@ from api import app
 
 
 @pytest.fixture(autouse=True)
-def bypass_x402_middleware() -> Iterator[None]:
+def bypass_x402_middleware() -> None:
     """Disable x402 payment gating so scraping endpoints can be tested."""
-    with patch(
-        "x402_middleware.X402Middleware.__call__",
-        lambda self, scope, receive, send: self.app(scope, receive, send),
-    ):
+    with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
         yield
 
 
 @pytest.fixture(autouse=True)
-def clear_response_cache() -> Iterator[None]:
+def clear_response_cache() -> None:
     """Clear the shared response cache so tests don't see each other's data."""
     from deps import cache as response_cache
 
@@ -75,10 +71,7 @@ async def test_v1_scrape_success(client: TestClient, mock_scrape_result: dict) -
 async def test_v1_scrape_with_json_schema(client: TestClient, mock_scrape_result: dict) -> None:
     schema = {"title": "page title"}
     extracted = {"title": "Example"}
-    with (
-        patch("routers.scraping.scraper") as mock_scraper,
-        patch("routers.scraping.extractor") as mock_extractor,
-    ):
+    with patch("routers.scraping.scraper") as mock_scraper, patch("routers.scraping.extractor") as mock_extractor:
         mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result)
         mock_extractor.extract = AsyncMock(return_value=extracted)
         resp = client.post("/v1/scrape", json={"url": "https://example.com", "jsonSchema": schema})
@@ -101,11 +94,7 @@ async def test_v1_detect_block_success(client: TestClient) -> None:
             )
         )
         mock_get_client.return_value = mock_client
-        resp = client.post(
-            "/v1/detect-block",
-            content='"https://example.com"',
-            headers={"content-type": "application/json"},
-        )
+        resp = client.post("/v1/detect-block", content="\"https://example.com\"", headers={"content-type": "application/json"})
 
     assert resp.status_code == 200
     data = resp.json()
@@ -132,9 +121,7 @@ async def test_v1_crawl_success(client: TestClient, mock_scrape_result: dict) ->
     pages = [mock_scrape_result]
     with patch("routers.scraping.scraper") as mock_scraper:
         mock_scraper.crawl = AsyncMock(return_value=pages)
-        resp = client.post(
-            "/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1}
-        )
+        resp = client.post("/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1})
 
     assert resp.status_code == 200
     data = resp.json()

tests/test_api_templates.py

@@ -5,7 +5,6 @@
 # Licensed under MIT.
 """End-to-end tests for the templates router endpoints."""
 
-from collections.abc import Iterator
 from unittest.mock import AsyncMock, patch
 
 import pytest
@@ -15,12 +14,9 @@ from api import app
 
 
 @pytest.fixture(autouse=True)
-def bypass_x402_middleware() -> Iterator[None]:
+def bypass_x402_middleware() -> None:
     """Disable x402 payment gating for template execution tests."""
-    with patch(
-        "x402_middleware.X402Middleware.__call__",
-        lambda self, scope, receive, send: self.app(scope, receive, send),
-    ):
+    with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)):
         yield
 
 
@@ -69,10 +65,7 @@ async def test_get_template_not_found(client: TestClient) -> None:
 async def test_execute_template(client: TestClient) -> None:
     result = {"success": True, "data": {"title": "Widget"}}
     with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)):
-        resp = client.post(
-            "/v1/templates/execute",
-            json={"template_id": "amazon_product", "url": "https://example.com"},
-        )
+        resp = client.post("/v1/templates/execute", json={"template_id": "amazon_product", "url": "https://example.com"})
 
     assert resp.status_code == 200
     data = resp.json()

tests/test_apify_schema.py

@@ -63,17 +63,10 @@ def test_array_and_object_fields() -> None:
 
 
 def test_nullable_field() -> None:
-    schema = (
-        ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema()
-    )
+    schema = ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema()
     assert schema["properties"]["optional"]["type"] == ["string", "null"]
 
 
 def test_prefab() -> None:
-    schema = (
-        ApifySchemaBuilder()
-        .set_prefab("START_URLS")
-        .add_string("url", "URL", required=True)
-        .build_input_schema()
-    )
+    schema = ApifySchemaBuilder().set_prefab("START_URLS").add_string("url", "URL", required=True).build_input_schema()
     assert schema["prefab"] == "START_URLS"

ultimate_scraper.py

@@ -314,11 +314,6 @@ class UltimateScraper:
 
                 driver = uc.Chrome(headless=True, use_subprocess=True)
                 driver.get(url)
-                # Note: time.sleep is used inside a sync thread-pool task because
-                # undetected_chromedriver's blocking API runs in asyncio.to_thread.
-                # Replacing with asyncio.sleep would require restructuring the tier
-                # to be fully async; this sync sleep is bounded and does not block
-                # the event loop directly.
                 time.sleep(random.uniform(2, 4))
                 html = driver.page_source
                 driver.quit()