STATUS.md

@@ -5,17 +5,17 @@
 > Where we are RIGHT NOW. Update before every commit.
 
 ## Last Updated
-2026-07-03 - Phase 0 router split + tests + Apify schema lib + PRY_API_KEY deployed
+2026-07-03 - Phase 0 infra hardening: metrics, redis/postgres, pinned deps, scrape metrics
 
 ## Current Status
-🟡 pre-production - core scraping/template routers split, e2e tests added, Talos PRY_API_KEY set, deploy pending.
+🟡 pre-production - router split deployed; infra hardening branch ready for review/deploy.
 
 ## Deployment Status
 - **Last deploy**: TBD
 - **Current version**: 3.0.0-phase0
 - **Health**: TBD
 - **Uptime (30d)**: TBD
-- **Active branch**: `main`
+- **Active branch**: `feat/phase0-rebased` (reconciled on top of latest `main`)
 
 ## Open Issues
 - _None — track in forgejo issues_
@@ -28,11 +28,23 @@
 - 2026-07-03: Added reusable Apify actor schema builder (`apify_schema.py`)
 - 2026-07-03: Set `PRY_API_KEY` in Talos `/srv/pry/.env`; auth now active on restart
 - 2026-07-03: Added `/v1/templates/batch` to x402 paid endpoints + pricing
+- 2026-07-03: Added `/metrics` endpoint + Prometheus counters/histograms for requests and scrapes
+- 2026-07-03: Added Redis + Postgres services to `docker-compose.yml`
+- 2026-07-03: Fixed Ollama URL to Talos (`100.104.130.92:11434`)
+- 2026-07-03: Added `LLMRegistry.complete_or_empty()` graceful fallback
+- 2026-07-03: Regenerated `requirements.txt`; added pinned `requirements.lock`
 
 ## Known Issues / Tech Debt
-- Deploy at /srv/pry/ is out of sync with repo (needs pull + restart to activate PRY_API_KEY and router changes)
+- Deploy at `/srv/pry/` is out of sync with repo (needs pull + restart to activate PRY_API_KEY, router changes, and metrics)
+- `feat/phase0-rebased` branch reconciles infra-hardening commits on top of latest `main`
+- pry-flaresolverr host port moved from 8191 to 8192 to avoid conflict with `rmi-flaresolverr`
 - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
-- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite.
+- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres.
+- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
+- License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
+- mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.
+- All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
+- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres.
 - PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
 - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
 - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.

api.py

@@ -47,6 +47,7 @@ from extractor import SchemaExtractor
 from mconfig import PryConfig
 from mcp_production import make_fallback_server, register_all
 from mcp_sse import mcp_post_message, mcp_sse_endpoint
+from observability import REQUEST_COUNT, REQUEST_LATENCY, get_metrics_output, setup_tracing
 from pipeline import HOOK_POINTS, get_pipeline, run_pipeline
 from pryextras import BatchProcessor, TransformEngine, recorder, streams
 from routers.auth import router as auth_router
@@ -80,6 +81,7 @@ except ImportError:
 async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """Startup: validate deps. Shutdown: cleanup clients."""
     logger.info("pry_startup", version="3.0.0")
+    setup_tracing()
     if not settings.api_key:
         logger.warning(
             "pry_api_key_unset",
@@ -252,8 +254,10 @@ class PryHttpMiddleware:
         start = time.time()
         request_id = self._get_header(scope, b"x-request-id") or uuid.uuid4().hex[:12]
         path = scope.get("path", "")
+        method = scope.get("method", "GET")
         client = scope.get("client")
         ip = client[0] if client else "unknown"
+        response_status = 200
 
         # Authentication check
         api_key = settings.api_key
@@ -327,7 +331,9 @@ class PryHttpMiddleware:
         request_id_bytes = request_id.encode()
 
         async def wrapped_send(message: dict[str, Any]) -> None:
+            nonlocal response_status
             if message["type"] == "http.response.start":
+                response_status = message.get("status", 200)
                 headers = list(message.get("headers", []))
                 headers.append((b"x-ratelimit-limit", ratelimit_limit))
                 headers.append((b"x-ratelimit-remaining", ratelimit_remaining))
@@ -339,12 +345,16 @@ class PryHttpMiddleware:
         await self.app(scope, receive, wrapped_send)
 
         elapsed = time.time() - start
+        if REQUEST_COUNT and REQUEST_LATENCY:
+            REQUEST_COUNT.labels(method=method, endpoint=path, status=str(response_status)).inc()
+            REQUEST_LATENCY.labels(endpoint=path).observe(elapsed)
         logger.info(
             "request_end",
             extra={
                 "request_id": request_id,
-                "method": scope.get("method", "GET"),
+                "method": method,
                 "path": path,
+                "status": response_status,
                 "duration": f"{elapsed:.3f}s",
             },
         )
@@ -2810,6 +2820,13 @@ app.include_router(health_router)
 app.include_router(scraping_router)
 app.include_router(templates_router)
 
+
+@app.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False)
+async def metrics() -> Response:
+    """Expose Prometheus metrics for scraping."""
+    data, content_type = get_metrics_output()
+    return Response(content=data, media_type=content_type)
+
 # ── Review ──
 
 

cli.py

@@ -83,6 +83,7 @@ def _spinner(label: str):
             if stop:
                 break
             print(f"  {CYAN}{c}{NC} {label}...  ", end="\r", flush=True)
+            # CLI spinner runs in a daemon thread; sync sleep is acceptable here.
             time.sleep(0.1)
 
     t = threading.Thread(target=_spin, daemon=True)

docker-compose.yml

@@ -80,3 +80,40 @@ services:
 volumes:
   pry-data:
   pry-sessions:
+
+  redis:
+    image: redis:7-alpine
+    container_name: pry-redis
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:6379:6379"
+    volumes:
+      - pry-redis:/data
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+          cpus: "0.5"
+
+  postgres:
+    image: postgres:15-alpine
+    container_name: pry-postgres
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - pry-postgres:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=pry
+      - POSTGRES_PASSWORD=pry
+      - POSTGRES_DB=pry
+    deploy:
+      resources:
+        limits:
+          memory: 512M
+          cpus: "0.5"
+
+volumes:
+  pry-sessions:
+  pry-redis:
+  pry-postgres:

llm_providers/registry.py

@@ -94,6 +94,35 @@ class LLMRegistry:
                     break
         raise Exception(f"All LLM providers failed. Last: {last_error}")
 
+    async def complete_or_empty(
+        self,
+        prompt: str,
+        system: str = "",
+        provider_name: str = "",
+        max_tokens: int = 1024,
+        temperature: float = 0.7,
+        model: str = "",
+        fallback: bool = True,
+    ) -> LLMResponse:
+        """Complete via LLM, returning an empty response on total failure.
+
+        Use this at API boundaries where a 500 is worse than degraded output.
+        """
+        try:
+            return await self.complete(
+                prompt, system, provider_name, max_tokens, temperature, model, fallback
+            )
+        except Exception as e:  # noqa: BLE001
+            logger.warning("llm_complete_failed_all_providers", extra={"error": str(e)[:120]})
+            return LLMResponse(
+                text="",
+                provider="none",
+                model="",
+                input_tokens=0,
+                output_tokens=0,
+                cost_usd=0.0,
+            )
+
     async def embed(self, text: str, provider_name: str = "", model: str = "") -> list[float]:
         names = [provider_name] if provider_name else list(self.fallback_chain)
         for name in names:

pyproject.toml

@@ -50,6 +50,9 @@ dependencies = [
     "structlog>=24.0.0",
     "sqlalchemy>=2.0.0",
     "aiosqlite>=0.19.0",
+    "prometheus-client>=0.21.0",
+    "opentelemetry-api>=1.29.0",
+    "opentelemetry-sdk>=1.29.0",
 ]
 
 [project.optional-dependencies]

requirements.lock

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Generated from pyproject.toml; pinned to currently installed versions.
+
+fastapi==0.128.8
+uvicorn==0.49.0
+trafilatura==2.1.0
+readability-lxml==0.8.4.1
+lxml==6.1.1
+httpx==0.28.1
+markdownify==1.2.3
+pydantic==2.12.5
+pydantic-settings==2.14.2
+playwright==1.60.0
+redis==8.0.0
+pypdf==6.14.2
+python-docx==1.2.0
+tiktoken==0.12.0
+pillow==12.2.0
+click==8.1.8
+pyyaml==6.0.3
+pandas==3.0.3
+anyio==4.14.1
+croniter==6.2.3
+structlog==26.1.0
+sqlalchemy==2.0.50
+aiosqlite==0.22.1
+prometheus-client==0.25.0
+opentelemetry-api==1.37.0
+opentelemetry-sdk==1.37.0

requirements.txt

@@ -27,3 +27,6 @@ structlog>=24.0.0
 sqlalchemy>=2.0.0
 aiosqlite>=0.19.0
 alembic>=1.14.0
+prometheus-client>=0.21.0
+opentelemetry-api>=1.29.0
+opentelemetry-sdk>=1.29.0

routers/scraping.py

@@ -32,6 +32,7 @@ from pydantic import BaseModel
 from client import get_client
 from deps import cache, extractor, queue, scraper
 from errors import ExternalServiceError, InvalidRequestError, PryError, ScrapeError
+from observability import track_scrape
 from scraper import BlockDetector
 from settings import settings
 
@@ -107,15 +108,16 @@ async def scrape(request: ScrapeRequest) -> dict[str, Any]:
         return cached
 
     try:
-        result = await scraper.scrape(
-            request.url,
-            {
-                "timeout": request.timeout,
-                "bypass_cloudflare": request.bypassCloudflare,
-                "js_render": request.jsRender,
-                "formats": request.formats,
-            },
-        )
+        with track_scrape(method="direct"):
+            result = await scraper.scrape(
+                request.url,
+                {
+                    "timeout": request.timeout,
+                    "bypass_cloudflare": request.bypassCloudflare,
+                    "js_render": request.jsRender,
+                    "formats": request.formats,
+                },
+            )
         if result.get("status") != "ok":
             raise ScrapeError(result.get("error", "Scrape failed"))
 
@@ -212,7 +214,8 @@ async def detect_lazy_content(
         generate_scroll_script,
     )
 
-    result = await scraper.scrape(url, {"bypass_cloudflare": True})
+    with track_scrape(method="lazy"):
+        result = await scraper.scrape(url, {"bypass_cloudflare": True})
     if result.get("status") != "ok":
         raise ScrapeError(result.get("error") or "Scrape failed")
 
@@ -255,14 +258,15 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]:
         task.add_done_callback(_log_crawl_job_failure)
         return {"success": True, "data": {"id": job_id, "status": "pending"}}
 
-    pages = await scraper.crawl(
-        request.url,
-        {
-            "max_pages": request.maxPages,
-            "max_depth": request.maxDepth,
-            "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
-        },
-    )
+    with track_scrape(method="crawl"):
+        pages = await scraper.crawl(
+            request.url,
+            {
+                "max_pages": request.maxPages,
+                "max_depth": request.maxDepth,
+                "timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
+            },
+        )
     return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}}
 
 
@@ -272,7 +276,8 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]:
 @router.post("/v1/map", summary="Discover URLs on a site")
 async def map_pages(request: MapRequest) -> dict[str, Any]:
     """Discover URLs on a site."""
-    urls = await scraper.map_urls(request.url, {"limit": request.limit})
+    with track_scrape(method="map"):
+        urls = await scraper.map_urls(request.url, {"limit": request.limit})
     return {"success": True, "data": {"links": urls}}
 
 
@@ -284,8 +289,9 @@ async def batch_scrape(urls: list[str] = Body(...), timeout: int = 30) -> dict[s
     """Scrape multiple URLs in parallel. Firecrawl charges extra for batch."""
     if len(urls) > 50:
         raise InvalidRequestError("Max 50 URLs per batch")
-    tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
-    results = await asyncio.gather(*tasks, return_exceptions=True)
+    with track_scrape(method="batch"):
+        tasks = [scraper.scrape(u, {"timeout": timeout, "bypass_cloudflare": True}) for u in urls]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
     pages = []
     for i, r in enumerate(results):
         if isinstance(r, BaseException):

settings.py

@@ -25,7 +25,7 @@ class PrySettings(BaseSettings):
     port: int = 8002
 
     # Ollama LLM endpoint
-    ollama_url: str = "http://100.100.18.18:11434"
+    ollama_url: str = "http://100.104.130.92:11434"
 
     # FlareSolverr Cloudflare bypass endpoint
     flaresolverr_url: str = "http://flaresolverr:8191/v1"

ultimate_scraper.py

@@ -314,6 +314,11 @@ class UltimateScraper:
 
                 driver = uc.Chrome(headless=True, use_subprocess=True)
                 driver.get(url)
+                # Note: time.sleep is used inside a sync thread-pool task because
+                # undetected_chromedriver's blocking API runs in asyncio.to_thread.
+                # Replacing with asyncio.sleep would require restructuring the tier
+                # to be fully async; this sync sleep is bounded and does not block
+                # the event loop directly.
                 time.sleep(random.uniform(2, 4))
                 html = driver.page_source
                 driver.quit()