diff --git a/=1.14.0 b/=1.14.0 deleted file mode 100644 index fe340f2..0000000 --- a/=1.14.0 +++ /dev/null @@ -1,15 +0,0 @@ -Collecting alembic - Using cached alembic-1.18.5-py3-none-any.whl.metadata (7.2 kB) -Requirement already satisfied: SQLAlchemy>=1.4.23 in /tmp/test-pry/lib/python3.12/site-packages (from alembic) (2.0.51) -Collecting Mako (from alembic) - Using cached mako-1.3.12-py3-none-any.whl.metadata (2.9 kB) -Requirement already satisfied: typing-extensions>=4.12 in /tmp/test-pry/lib/python3.12/site-packages (from alembic) (4.16.0) -Requirement already satisfied: greenlet>=1 in /tmp/test-pry/lib/python3.12/site-packages (from SQLAlchemy>=1.4.23->alembic) (3.5.3) -Collecting MarkupSafe>=0.9.2 (from Mako->alembic) - Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl.metadata (2.7 kB) -Using cached alembic-1.18.5-py3-none-any.whl (264 kB) -Using cached mako-1.3.12-py3-none-any.whl (78 kB) -Using cached markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl (22 kB) -Installing collected packages: MarkupSafe, Mako, alembic - -Successfully installed Mako-1.3.12 MarkupSafe-3.0.3 alembic-1.18.5 diff --git a/Dockerfile b/Dockerfile index db650c9..e6d69bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ COPY routers/ ./routers/ COPY llm_providers/ ./llm_providers/ COPY stealth_scripts/ ./stealth_scripts/ COPY templates/ ./templates/ -RUN mkdir -p /app/sessions +RUN mkdir -p /app/sessions /root/.pry EXPOSE 8002 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \ diff --git a/STATUS.md b/STATUS.md index e6cb1e4..37cc0c0 100644 --- a/STATUS.md +++ b/STATUS.md @@ -5,17 +5,17 @@ > Where we are RIGHT NOW. Update before every commit. ## Last Updated -2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed +2026-07-03 - Phase 0 infra hardening: metrics, redis/postgres, pinned deps, scrape metrics ## Current Status -🟡 pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending. +🟡 pre-production - router split deployed; infra hardening branch ready for review/deploy. ## Deployment Status -- **Last deploy**: 2026-07-03 +- **Last deploy**: TBD - **Current version**: 3.0.0-phase0 -- **Health**: 🟢 healthy +- **Health**: TBD - **Uptime (30d)**: TBD -- **Active branch**: `feat/phase0-router-split` (PR #5 awaiting review) +- **Active branch**: `feat/phase0-rebased` (reconciled on top of latest `main`) ## Open Issues - _None — track in forgejo issues_ @@ -27,15 +27,24 @@ - 2026-07-03: Added e2e tests for scraping and template routers (23 new/passing) - 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`) - 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart -- 2026-07-03: Registered `/v1/templates/batch` as paid x402 operation -- 2026-07-03: Deployed to Talos `/srv/pry/`; pry container healthy on 127.0.0.1:8005 -- 2026-07-03: Opened PR #5: https://git.rugmunch.io/RugMunchMedia/pryscraper/pulls/5 +- 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing +- 2026-07-03: Added `/metrics` endpoint + Prometheus counters/histograms for requests and scrapes +- 2026-07-03: Added Redis + Postgres services to `docker-compose.yml` +- 2026-07-03: Fixed Ollama URL to Talos (`100.104.130.92:11434`) +- 2026-07-03: Added `LLMRegistry.complete_or_empty()` graceful fallback +- 2026-07-03: Regenerated `requirements.txt`; added pinned `requirements.lock` ## Known Issues / Tech Debt -- PR #5 must be reviewed/merged to bring `main` in sync with deployed code +- Deploy at `/srv/pry/` is out of sync with repo (needs pull + restart to activate PRY_API_KEY, router changes, and metrics) +- `feat/phase0-rebased` branch reconciles infra-hardening commits on top of latest `main` - pry-flaresolverr host port moved from 8191 to 8192 to avoid conflict with `rmi-flaresolverr` - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage. -- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite. +- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres. +- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active. +- License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002. +- mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy. +- All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage. +- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres. - PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active. - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002. - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy. diff --git a/alembic/env.py b/alembic/env.py index b277d38..e260ebe 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -4,16 +4,15 @@ from logging.config import fileConfig from sqlalchemy import engine_from_config, pool from alembic import context -from db import Base +from db import Base, _resolve_database_url config = context.config if config.config_file_name is not None: fileConfig(config.config_file_name) -db_url = os.getenv("PRY_DATABASE_URL") -if db_url is not None: - config.set_main_option("sqlalchemy.url", db_url) +# Always use the same URL resolution logic as the application. +config.set_main_option("sqlalchemy.url", os.getenv("PRY_DATABASE_URL") or _resolve_database_url()) target_metadata = Base.metadata exclude_tables = {"spatial_ref_sys"} diff --git a/alembic/versions/0001_initial_schema.py b/alembic/versions/0001_initial_schema.py index 11ef9b9..0b95a93 100644 --- a/alembic/versions/0001_initial_schema.py +++ b/alembic/versions/0001_initial_schema.py @@ -5,6 +5,7 @@ Revises: Create Date: 2026-07-03 00:00:00.000000 """ + from __future__ import annotations from collections.abc import Sequence @@ -58,7 +59,9 @@ def upgrade() -> None: sa.Column("ts", sa.DateTime(), index=True), sa.PrimaryKeyConstraint("id"), ) - op.create_index("ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")]) + op.create_index( + "ix_intel_competitor_ts", "intel_snapshots", ["competitor_id", sa.text("ts DESC")] + ) op.create_table( "costing_entries", sa.Column("id", sa.Integer(), nullable=False), diff --git a/api.py b/api.py index ee377fb..ffc08e4 100644 --- a/api.py +++ b/api.py @@ -27,6 +27,7 @@ from errors import ( ) from mcp_production import make_fallback_server, register_all from mcp_sse import mcp_post_message, mcp_sse_endpoint +from observability import REQUEST_COUNT, REQUEST_LATENCY, setup_tracing from pipeline import get_pipeline from routers.advanced import router as advanced_router from routers.agency import router as agency_router @@ -105,8 +106,12 @@ except ImportError: async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Startup: validate deps. Shutdown: cleanup clients.""" logger.info("pry_startup", version="3.0.0") + setup_tracing() if not settings.api_key: - logger.warning("pry_api_key_unset", message="PRY_API_KEY is not set; all protected endpoints are publicly accessible") + logger.warning( + "pry_api_key_unset", + message="PRY_API_KEY is not set; all protected endpoints are publicly accessible", + ) get_pipeline() # Initialize pipeline yield logger.info("pry_shutdown") @@ -274,8 +279,10 @@ class PryHttpMiddleware: start = time.time() request_id = self._get_header(scope, b"x-request-id") or uuid.uuid4().hex[:12] path = scope.get("path", "") + method = scope.get("method", "GET") client = scope.get("client") ip = client[0] if client else "unknown" + response_status = 200 # Authentication check api_key = settings.api_key @@ -349,7 +356,9 @@ class PryHttpMiddleware: request_id_bytes = request_id.encode() async def wrapped_send(message: dict[str, Any]) -> None: + nonlocal response_status if message["type"] == "http.response.start": + response_status = message.get("status", 200) headers = list(message.get("headers", [])) headers.append((b"x-ratelimit-limit", ratelimit_limit)) headers.append((b"x-ratelimit-remaining", ratelimit_remaining)) @@ -361,12 +370,16 @@ class PryHttpMiddleware: await self.app(scope, receive, wrapped_send) elapsed = time.time() - start + if REQUEST_COUNT and REQUEST_LATENCY: + REQUEST_COUNT.labels(method=method, endpoint=path, status=str(response_status)).inc() + REQUEST_LATENCY.labels(endpoint=path).observe(elapsed) logger.info( "request_end", extra={ "request_id": request_id, - "method": scope.get("method", "GET"), + "method": method, "path": path, + "status": response_status, "duration": f"{elapsed:.3f}s", }, ) @@ -719,6 +732,8 @@ app.include_router(vision_router) app.include_router(webhooks_router) app.include_router(x402_router) + + # ── Review ── diff --git a/apify_schema.py b/apify_schema.py index 0d6cc84..5f72f39 100644 --- a/apify_schema.py +++ b/apify_schema.py @@ -231,7 +231,9 @@ class ObjectField(FieldSpec): class ApifySchemaBuilder: """Fluent builder for Apify actor input/output schemas.""" - def __init__(self, title: str = "Actor Input", description: str = "", schema_version: int = 1) -> None: + def __init__( + self, title: str = "Actor Input", description: str = "", schema_version: int = 1 + ) -> None: self.title = title self.description = description self.schema_version = schema_version @@ -284,7 +286,13 @@ class ApifySchemaBuilder: maximum: int | None = None, ) -> ApifySchemaBuilder: field = IntegerField( - name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum + name, + title, + description=description, + default=default, + nullable=nullable, + minimum=minimum, + maximum=maximum, ) return self.add_field(field, required=required) @@ -300,7 +308,13 @@ class ApifySchemaBuilder: maximum: float | None = None, ) -> ApifySchemaBuilder: field = NumberField( - name, title, description=description, default=default, nullable=nullable, minimum=minimum, maximum=maximum + name, + title, + description=description, + default=default, + nullable=nullable, + minimum=minimum, + maximum=maximum, ) return self.add_field(field, required=required) @@ -313,7 +327,9 @@ class ApifySchemaBuilder: default: bool | None = None, nullable: bool = False, ) -> ApifySchemaBuilder: - field = BooleanField(name, title, description=description, default=default, nullable=nullable) + field = BooleanField( + name, title, description=description, default=default, nullable=nullable + ) return self.add_field(field, required=required) def add_enum( @@ -326,7 +342,9 @@ class ApifySchemaBuilder: default: str | None = None, nullable: bool = False, ) -> ApifySchemaBuilder: - field = EnumField(name, title, enum, description=description, default=default, nullable=nullable) + field = EnumField( + name, title, enum, description=description, default=default, nullable=nullable + ) return self.add_field(field, required=required) def add_array( @@ -365,7 +383,13 @@ class ApifySchemaBuilder: nested_required: list[str] | None = None, ) -> ApifySchemaBuilder: field = ObjectField( - name, title, properties, description=description, default=default, nullable=nullable, required=nested_required + name, + title, + properties, + description=description, + default=default, + nullable=nullable, + required=nested_required, ) return self.add_field(field, required=required) diff --git a/cli.py b/cli.py index c51e59b..9f9d976 100755 --- a/cli.py +++ b/cli.py @@ -83,6 +83,7 @@ def _spinner(label: str): if stop: break print(f" {CYAN}{c}{NC} {label}... ", end="\r", flush=True) + # CLI spinner runs in a daemon thread; sync sleep is acceptable here. time.sleep(0.1) t = threading.Thread(target=_spin, daemon=True) @@ -196,6 +197,7 @@ def cmd_screenshot(url, output=None, timeout=30): def cmd_migrate(): """Run pending database migrations (alembic upgrade head).""" from db import run_migrations + run_migrations() print(f"{GREEN}✓{NC} Migrations up to date") diff --git a/docker-compose.yml b/docker-compose.yml index 48da537..d083265 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ services: ports: - "127.0.0.1:8005:8002" volumes: + - pry-data:/root/.pry - pry-sessions:/app/sessions env_file: .env environment: @@ -77,4 +78,42 @@ services: - tor volumes: + pry-data: pry-sessions: + + redis: + image: redis:7-alpine + container_name: pry-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + volumes: + - pry-redis:/data + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + + postgres: + image: postgres:15-alpine + container_name: pry-postgres + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" + volumes: + - pry-postgres:/var/lib/postgresql/data + environment: + - POSTGRES_USER=pry + - POSTGRES_PASSWORD=pry + - POSTGRES_DB=pry + deploy: + resources: + limits: + memory: 512M + cpus: "0.5" + +volumes: + pry-sessions: + pry-redis: + pry-postgres: diff --git a/llm_providers/registry.py b/llm_providers/registry.py index f4cee3d..579cf0d 100644 --- a/llm_providers/registry.py +++ b/llm_providers/registry.py @@ -94,6 +94,35 @@ class LLMRegistry: break raise Exception(f"All LLM providers failed. Last: {last_error}") + async def complete_or_empty( + self, + prompt: str, + system: str = "", + provider_name: str = "", + max_tokens: int = 1024, + temperature: float = 0.7, + model: str = "", + fallback: bool = True, + ) -> LLMResponse: + """Complete via LLM, returning an empty response on total failure. + + Use this at API boundaries where a 500 is worse than degraded output. + """ + try: + return await self.complete( + prompt, system, provider_name, max_tokens, temperature, model, fallback + ) + except Exception as e: # noqa: BLE001 + logger.warning("llm_complete_failed_all_providers", extra={"error": str(e)[:120]}) + return LLMResponse( + text="", + provider="none", + model="", + input_tokens=0, + output_tokens=0, + cost_usd=0.0, + ) + async def embed(self, text: str, provider_name: str = "", model: str = "") -> list[float]: names = [provider_name] if provider_name else list(self.fallback_chain) for name in names: diff --git a/pyproject.toml b/pyproject.toml index ccc695c..efab7e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,9 @@ dependencies = [ "structlog>=24.0.0", "sqlalchemy>=2.0.0", "aiosqlite>=0.19.0", + "prometheus-client>=0.21.0", + "opentelemetry-api>=1.29.0", + "opentelemetry-sdk>=1.29.0", ] [project.optional-dependencies] diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..3805281 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Generated from pyproject.toml; pinned to currently installed versions. + +fastapi==0.128.8 +uvicorn==0.49.0 +trafilatura==2.1.0 +readability-lxml==0.8.4.1 +lxml==6.1.1 +httpx==0.28.1 +markdownify==1.2.3 +pydantic==2.12.5 +pydantic-settings==2.14.2 +playwright==1.60.0 +redis==8.0.0 +pypdf==6.14.2 +python-docx==1.2.0 +tiktoken==0.12.0 +pillow==12.2.0 +click==8.1.8 +pyyaml==6.0.3 +pandas==3.0.3 +anyio==4.14.1 +croniter==6.2.3 +structlog==26.1.0 +sqlalchemy==2.0.50 +aiosqlite==0.22.1 +prometheus-client==0.25.0 +opentelemetry-api==1.37.0 +opentelemetry-sdk==1.37.0 diff --git a/requirements.txt b/requirements.txt index 966ffe3..ad5d64e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,6 @@ structlog>=24.0.0 sqlalchemy>=2.0.0 aiosqlite>=0.19.0 alembic>=1.14.0 +prometheus-client>=0.21.0 +opentelemetry-api>=1.29.0 +opentelemetry-sdk>=1.29.0 diff --git a/routers/scraping.py b/routers/scraping.py index 1d7250d..0f9d052 100644 --- a/routers/scraping.py +++ b/routers/scraping.py @@ -32,6 +32,7 @@ from pydantic import BaseModel from client import get_client from deps import cache, extractor, queue, scraper from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError +from observability import track_scrape from scraper import BlockDetector from settings import settings @@ -107,15 +108,16 @@ async def scrape(request: ScrapeRequest) -> dict[str, Any]: return cached try: - result = await scraper.scrape( - request.url, - { - "timeout": request.timeout, - "bypass_cloudflare": request.bypassCloudflare, - "js_render": request.jsRender, - "formats": request.formats, - }, - ) + with track_scrape(method="direct"): + result = await scraper.scrape( + request.url, + { + "timeout": request.timeout, + "bypass_cloudflare": request.bypassCloudflare, + "js_render": request.jsRender, + "formats": request.formats, + }, + ) if result.get("status") != "ok": raise ScrapeError(result.get("error", "Scrape failed")) @@ -212,7 +214,8 @@ async def detect_lazy_content( generate_scroll_script, ) - result = await scraper.scrape(url, {"bypass_cloudflare": True}) + with track_scrape(method="lazy"): + result = await scraper.scrape(url, {"bypass_cloudflare": True}) if result.get("status") != "ok": raise ScrapeError(result.get("error") or "Scrape failed") @@ -255,14 +258,15 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]: task.add_done_callback(_log_crawl_job_failure) return {"success": True, "data": {"id": job_id, "status": "pending"}} - pages = await scraper.crawl( - request.url, - { - "max_pages": request.maxPages, - "max_depth": request.maxDepth, - "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, - }, - ) + with track_scrape(method="crawl"): + pages = await scraper.crawl( + request.url, + { + "max_pages": request.maxPages, + "max_depth": request.maxDepth, + "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60, + }, + ) return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}} @@ -272,7 +276,8 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]: @router.post("/v1/map", summary="Discover URLs on a site") async def map_pages(request: MapRequest) -> dict[str, Any]: """Discover URLs on a site.""" - urls = await scraper.map_urls(request.url, {"limit": request.limit}) + with track_scrape(method="map"): + urls = await scraper.map_urls(request.url, {"limit": request.limit}) return {"success": True, "data": {"links": urls}} @@ -284,8 +289,9 @@ async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[s """Scrape multiple URLs in parallel. Firecrawl charges extra for batch.""" if len(urls) > 50: raise InvalidRequestError("Max 50 URLs per batch") - tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] - results = await asyncio.gather(*tasks, return_exceptions=True) + with track_scrape(method="batch"): + tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls] + results = await asyncio.gather(*tasks, return_exceptions=True) pages = [] for i, r in enumerate(results): if isinstance(r, BaseException): diff --git a/routers/system.py b/routers/system.py index 95bf02c..781035d 100644 --- a/routers/system.py +++ b/routers/system.py @@ -10,13 +10,22 @@ from __future__ import annotations import logging -from fastapi import APIRouter +from fastapi import APIRouter, Response from fastapi.responses import JSONResponse +from observability import get_metrics_output + logger = logging.getLogger(__name__) router = APIRouter(tags=["System"]) +@router.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False) +async def metrics() -> Response: + """Expose Prometheus metrics for scraping.""" + data, content_type = get_metrics_output() + return Response(content=data, media_type=content_type) + + @router.get( "/openapi.json", tags=["System"], diff --git a/settings.py b/settings.py index e4ed772..8f432ff 100644 --- a/settings.py +++ b/settings.py @@ -25,7 +25,7 @@ class PrySettings(BaseSettings): port: int = 8002 # Ollama LLM endpoint - ollama_url: str = "http://100.100.18.18:11434" + ollama_url: str = "http://100.104.130.92:11434" # FlareSolverr Cloudflare bypass endpoint flaresolverr_url: str = "http://flaresolverr:8191/v1" diff --git a/tests/test_api.py b/tests/test_api.py index 39b998b..595c343 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -47,11 +47,17 @@ async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> Non """Protected endpoints require a valid Bearer PRY_API_KEY.""" monkeypatch.setattr("api.settings.api_key", "secret-pry-key") # Disable x402 payment gating so we can test auth in isolation. - with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): resp = client.post("/v1/scrape", json={"url": "https://example.com"}) assert resp.status_code == 401 - with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): resp = client.post( "/v1/scrape", json={"url": "https://example.com"}, @@ -59,7 +65,10 @@ async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> Non ) assert resp.status_code == 401 - with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): resp = client.post( "/v1/scrape", json={"url": "https://example.com"}, @@ -67,4 +76,3 @@ async def test_api_key_required_when_set(client: TestClient, monkeypatch) -> Non ) # auth passes; endpoint returns 402 because we did not mock scraper, which is fine assert resp.status_code in (200, 402) - diff --git a/tests/test_api_scraping.py b/tests/test_api_scraping.py index 29eccc8..1d15549 100644 --- a/tests/test_api_scraping.py +++ b/tests/test_api_scraping.py @@ -11,6 +11,7 @@ TestClient. External I/O (HTTP requests, browser automation) is mocked at the deterministic. """ +from collections.abc import Iterator from unittest.mock import AsyncMock, patch import pytest @@ -20,14 +21,17 @@ from api import app @pytest.fixture(autouse=True) -def bypass_x402_middleware() -> None: +def bypass_x402_middleware() -> Iterator[None]: """Disable x402 payment gating so scraping endpoints can be tested.""" - with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): yield @pytest.fixture(autouse=True) -def clear_response_cache() -> None: +def clear_response_cache() -> Iterator[None]: """Clear the shared response cache so tests don't see each other's data.""" from deps import cache as response_cache @@ -71,7 +75,10 @@ async def test_v1_scrape_success(client: TestClient, mock_scrape_result: dict) - async def test_v1_scrape_with_json_schema(client: TestClient, mock_scrape_result: dict) -> None: schema = {"title": "page title"} extracted = {"title": "Example"} - with patch("routers.scraping.scraper") as mock_scraper, patch("routers.scraping.extractor") as mock_extractor: + with ( + patch("routers.scraping.scraper") as mock_scraper, + patch("routers.scraping.extractor") as mock_extractor, + ): mock_scraper.scrape = AsyncMock(return_value=mock_scrape_result) mock_extractor.extract = AsyncMock(return_value=extracted) resp = client.post("/v1/scrape", json={"url": "https://example.com", "jsonSchema": schema}) @@ -94,7 +101,11 @@ async def test_v1_detect_block_success(client: TestClient) -> None: ) ) mock_get_client.return_value = mock_client - resp = client.post("/v1/detect-block", content="\"https://example.com\"", headers={"content-type": "application/json"}) + resp = client.post( + "/v1/detect-block", + content='"https://example.com"', + headers={"content-type": "application/json"}, + ) assert resp.status_code == 200 data = resp.json() @@ -121,7 +132,9 @@ async def test_v1_crawl_success(client: TestClient, mock_scrape_result: dict) -> pages = [mock_scrape_result] with patch("routers.scraping.scraper") as mock_scraper: mock_scraper.crawl = AsyncMock(return_value=pages) - resp = client.post("/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1}) + resp = client.post( + "/v1/crawl", json={"url": "https://example.com", "maxPages": 2, "maxDepth": 1} + ) assert resp.status_code == 200 data = resp.json() diff --git a/tests/test_api_templates.py b/tests/test_api_templates.py index 17a571a..6b23efa 100644 --- a/tests/test_api_templates.py +++ b/tests/test_api_templates.py @@ -5,6 +5,7 @@ # Licensed under MIT. """End-to-end tests for the templates router endpoints.""" +from collections.abc import Iterator from unittest.mock import AsyncMock, patch import pytest @@ -14,9 +15,12 @@ from api import app @pytest.fixture(autouse=True) -def bypass_x402_middleware() -> None: +def bypass_x402_middleware() -> Iterator[None]: """Disable x402 payment gating for template execution tests.""" - with patch("x402_middleware.X402Middleware.__call__", lambda self, scope, receive, send: self.app(scope, receive, send)): + with patch( + "x402_middleware.X402Middleware.__call__", + lambda self, scope, receive, send: self.app(scope, receive, send), + ): yield @@ -65,7 +69,10 @@ async def test_get_template_not_found(client: TestClient) -> None: async def test_execute_template(client: TestClient) -> None: result = {"success": True, "data": {"title": "Widget"}} with patch("routers.templates.execute_template", new=AsyncMock(return_value=result)): - resp = client.post("/v1/templates/execute", json={"template_id": "amazon_product", "url": "https://example.com"}) + resp = client.post( + "/v1/templates/execute", + json={"template_id": "amazon_product", "url": "https://example.com"}, + ) assert resp.status_code == 200 data = resp.json() diff --git a/tests/test_apify_schema.py b/tests/test_apify_schema.py index 56fe1b5..80a0848 100644 --- a/tests/test_apify_schema.py +++ b/tests/test_apify_schema.py @@ -63,10 +63,17 @@ def test_array_and_object_fields() -> None: def test_nullable_field() -> None: - schema = ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema() + schema = ( + ApifySchemaBuilder().add_string("optional", "Optional", nullable=True).build_input_schema() + ) assert schema["properties"]["optional"]["type"] == ["string", "null"] def test_prefab() -> None: - schema = ApifySchemaBuilder().set_prefab("START_URLS").add_string("url", "URL", required=True).build_input_schema() + schema = ( + ApifySchemaBuilder() + .set_prefab("START_URLS") + .add_string("url", "URL", required=True) + .build_input_schema() + ) assert schema["prefab"] == "START_URLS" diff --git a/ultimate_scraper.py b/ultimate_scraper.py index a4c8147..7ef4015 100644 --- a/ultimate_scraper.py +++ b/ultimate_scraper.py @@ -314,6 +314,11 @@ class UltimateScraper: driver = uc.Chrome(headless=True, use_subprocess=True) driver.get(url) + # Note: time.sleep is used inside a sync thread-pool task because + # undetected_chromedriver's blocking API runs in asyncio.to_thread. + # Replacing with asyncio.sleep would require restructuring the tier + # to be fully async; this sync sleep is bounded and does not block + # the event loop directly. time.sleep(random.uniform(2, 4)) html = driver.page_source driver.quit()