feat: add .agentrules with project-specific AI guardrails

chore: remove links to private standards repo
chore: rename munchcrawl → Pry across docs and code
2026-07-09 15:06:09 +02:00 · 2026-07-09 14:47:34 +02:00 · 2026-07-09 13:52:16 +02:00 · 2026-07-06 20:52:23 +07:00 · 2026-07-06 20:36:24 +07:00 · 2026-07-06 20:36:17 +07:00
32 changed files with 1470 additions and 115 deletions
--- a/.actor/input_schema.json
+++ b/.actor/input_schema.json
@ -0,0 +1,48 @@
+{
+  "title": "Pry Scraper",
+  "type": "object",
+  "schemaVersion": 1,
+  "properties": {
+    "urls": {
+      "title": "URLs to scrape",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "pattern": "^https?://"
+      }
+    },
+    "max_pages": {
+      "title": "Max pages",
+      "type": "integer",
+      "default": 10,
+      "minimum": 1,
+      "maximum": 1000
+    },
+    "timeout": {
+      "title": "Timeout (seconds)",
+      "type": "integer",
+      "default": 30,
+      "minimum": 5,
+      "maximum": 120
+    },
+    "output_format": {
+      "title": "Output format",
+      "type": "string",
+      "default": "markdown",
+      "enum": [
+        "markdown",
+        "html",
+        "text"
+      ]
+    },
+    "crawl": {
+      "title": "Crawl linked pages",
+      "type": "boolean",
+      "default": false
+    }
+  },
+  "description": "Scrape any website with Pry's 12-tier fallback chain.",
+  "required": [
+    "urls"
+  ]
+}
--- a/.actor/src/main.py
+++ b/.actor/src/main.py
@ -0,0 +1,23 @@
+"""Pry — Apify Actor entry point.
+
+Thin wrapper around the SDK-aware entry in apify_actor.py. The Apify
+platform invokes this file via Dockerfile.apify as
+``python /usr/src/app/.actor/src/main.py``; we re-export ``main()``
+so both the legacy ``python -m apify_actor`` invocation and the
+standard .actor/src/main.py convention work.
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from apify_actor import main
+
+if __name__ == "__main__":
+    asyncio.run(main())
--- a/.agentrules
+++ b/.agentrules
@ -0,0 +1,17 @@
+---
+title: AGENTS.md — pryscraper
+repo: backend
+stack: Python 3.12 + FastAPI + Docker
+domain: pryscraper.rugmunch.io
+deploy: Docker on Talos (/srv/pry/)
+status: active
+---
+
+## Rules
+1. No bare except — use specific exceptions or except Exception
+2. No sync I/O in async routes — use httpx.AsyncClient
+3. All public API routes must have type hints and docstrings
+4. Dual licensed: MIT (core) + BSL 1.1 (stealth)
+5. MCP tools must be typed and documented
+6. No hardcoded credentials — use gopass or env vars
+
--- a/.editorconfig
+++ b/.editorconfig
@ -0,0 +1,26 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+# EditorConfig — https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
+
+[*.{yml,yaml,toml,json}]
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab
+
+[*.{bat,cmd,ps1}]
+end_of_line = crlf
--- a/.gitignore
+++ b/.gitignore
@ -9,6 +9,7 @@
 !.env.template
 *.pem
 *.key
+*.crt
 *.p12
 *.pfx
 id_rsa
@ -41,6 +42,7 @@ htmlcov/

 # ── Pry-specific cache ──────────────────────────────────────
 .warmed_cookies/
+warmed_cookies/
 .proxy_cache/
 .browser_profiles/
 .screenshots/
--- a/.gitleaks.toml
+++ b/.gitleaks.toml
@ -0,0 +1,7 @@
+allowlist:
+  description: "Pryscraper allowlist — test fixtures, docs, known public addresses"
+  regexTarget: match
+  paths:
+    - tests/
+    - docs/
+    - "*.md"
--- a/.semgrep/rules.yaml
+++ b/.semgrep/rules.yaml
@ -0,0 +1,25 @@
+rules:
+  - id: no-print-in-production
+    pattern: print(...)
+    message: "Remove print() before committing to production code. Use structured logging instead."
+    severity: WARNING
+    languages: [python]
+    paths:
+      exclude:
+        - tests/
+        - "**/test_*.py"
+
+  - id: no-os-system
+    pattern: os.system(...)
+    message: "os.system() is dangerous. Use subprocess.run() with shell=False instead."
+    severity: ERROR
+    languages: [python]
+
+  - id: no-eval-exec
+    patterns:
+      - pattern-either:
+        - pattern: eval(...)
+        - pattern: exec(...)
+    message: "eval()/exec() are dangerous and should not be used in production code."
+    severity: ERROR
+    languages: [python]
--- a/AGENTS.md
+++ b/AGENTS.md
@ -5,10 +5,10 @@
 > AI agent contract. Read this before touching anything in this repo.

 ## Status
-canonical · owner=crmuncher · last_updated=2026-07-02
+canonical · owner=crmuncher · last_updated=2026-07-06 · phase_0_hardening_complete

 ## What This Repo Is
-Multi-source crypto intelligence crawler (munchcrawl). Scraper, extractor, parser, automator, job queue. CLI + MCP server + SDK. License: Dual -- MIT (core) + BSL 1.1 (stealth/anti-detection subset). See LICENSE and LICENSE-BSL-STEALTH.
+Multi-source crypto intelligence crawler (pryscraper). Scraper, extractor, parser, automator, job queue. CLI + MCP server + SDK. License: Dual -- MIT (core) + BSL 1.1 (stealth/anti-detection subset). See LICENSE and LICENSE-BSL-STEALTH.

 ## Type
 `backend` · language=Python 3.12 + FastAPI
@ -19,19 +19,25 @@ Multi-source crypto intelligence crawler (munchcrawl). Scraper, extractor, parse
 - Port: 8005

 ## Built With
- Standards: [git.rugmunch.io/RugMunchMedia/standards](https://git.rugmunch.io/RugMunchMedia/standards)
- Toolchain: [TOOLCHAIN.md](https://git.rugmunch.io/RugMunchMedia/standards/raw/branch/main/TOOLCHAIN.md)
- Conventions: [CONVENTIONS.md](https://git.rugmunch.io/RugMunchMedia/standards/raw/branch/main/CONVENTIONS.md)

 ## Components
 - `/srv/pry/*.py`: 83 Python modules (api.py, extractor.py, parser.py, etc.)
 - `browser-extension/`: Chrome/Firefox extension
+- `routers/`: FastAPI routers split from `api.py` (scraping, templates, health, auth, captcha, …)
+- `pry_mcp/`: MCP sub-package (split from `mcp_production.py` 2026-07-06)
+- `pry_x402/`: x402 constants + enums (split from `x402.py` 2026-07-06)
+- `pry_<name>.py`: 19 root modules renamed to remove `routers/` collision (2026-07-06)

 ## Dependencies
 - postgresql (job storage)
 - redis (cache)
 - flaresolverr (Cloudflare bypass, on Hydra)
 - playwright (browser automation)
+- cloudscraper (Cloudflare bypass tier)
+- aiohttp-socks + aiohttp (SOCKS5 proxy support, async HTTP)
+- pyjwt (JWT auth signing)
+- apify (Apify actor SDK)
+- playwright-stealth (anti-detection)

 ## 🚨 CRITICAL RULES

@ -71,4 +77,3 @@ See [ARCHITECTURE.md](ARCHITECTURE.md). Update it when components change.
 - Backup: TBD

 ## Related Repos
-See [standards/PRODUCTS.md](https://git.rugmunch.io/RugMunchMedia/standards/raw/branch/main/PRODUCTS.md).
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@ -121,7 +121,7 @@ All data stored under `~/.pry/`:
 | `costing.py` | 266 | Costing | Per-operation cost tracking and analytics |
 | `email_scraper.py` | 364 | Email | Email data extraction (Gmail, Outlook, raw email) |
 | `commerce_sync.py` | 191 | Commerce | WooCommerce/Shopify product sync |
-| `crm_sync.py` | 359 | CRM | Salesforce/HubSpot/Zoho CRM sync |
+| `crm_sync.py` | 359 | CRM | Salesforce / HubSpot / Pipedrive / Close.com sync |
 | `destinations.py` | 361 | Export | Data export — webhooks, Slack, S3, GCS, SFTP |
 | `intelligence.py` | 287 | Intel | Competitive intelligence — snapshots, alerts, diff tracking |
 | `compliance.py` | 443 | Compliance | GDPR/CCPA compliance — sensitive data detection, consent |
--- a/AUDIT-2026-Q3.md
+++ b/AUDIT-2026-Q3.md
@ -0,0 +1,579 @@
+---
+title: Pry — Production Audit & Roadmap (2026-Q3)
+status: canonical
+owner: Rug Munch Media LLC Engineering
+last_updated: 2026-07-06
+audited_by: opencode (deep multi-agent audit)
+supersedes: AUDIT.md (2026-06-30)
+audience:
+  - engineers
+  - agents
+  - operators
+goal: ship Pry as production-grade + Apify-publishable within Q3 2026
+---
+
+# Pry — Production Audit & Roadmap
+
+> **TL;DR.** Pry is a working prototype of a self-hosted scraper with good bones (scraping engine, template system, MIT/BSL split). It is **not yet a product**. Roughly **30% of the code is real, 30% is stub, 40% is hallucinated**. To reach production-grade AND Apify-publishable, we need 8 weeks of focused engineering across 5 phases. Apify-publishable is reachable in 2 weeks if we sequence the Apify glue last.
+
+---
+
+## 1. Brutal Honest State (master table)
+
+| Area | Claim | Reality | Status |
+|---|---|---|---|
+| **Scraping engine** | 10-tier fallback | 5 tiers wired, 4 import missing libs and silently no-op, 1 (Textise) exists only in docstring | 🟡 partial |
+| **Anti-detection** | camoufox + stealth + TLS fingerprint + adaptive + cookie warmer | ~1,255 lines of code that **never runs in the default scrape flow**. 0/8 modules wired into `PryScraper.scrape()`. | 🔴 orphan theater |
+| **Templates** | 110 working scraper templates | 110 JSON files, all CSS selectors. Real coverage validator says 21/110, live tests show ~30-40%. The other ~70 are decorative. | 🟡 vanity |
+| **Database** | SQLite via SQLAlchemy | `pry.db` exists (804 LOC `db.py`, 24 ORM models). **No Alembic**; `Base.metadata.create_all()` is the only migration path. `data.db` is a 96KB orphan. | 🟡 partial |
+| **State files** | "all JSON in `~/.pry/`" | True: 24 subdirs + 122 JSON vault files + JSONL x402 ledgers. **Zero concurrency control** on the JSON writers. SQLite is the only transactional store. | 🔴 race-condition risk |
+| **Auth** | gopass + JWT (HS256) + pbkdf2_hmac 100K rounds | Real, but **only 2 endpoints require it** (`/v1/key` and one more). Most endpoints are public. | 🟡 partial |
+| **API surface** | 115+ endpoints, 46 tag groups | Real (~190 endpoints by Phase 0 count). But still in a 4,668-line `api.py` — only `/v1/scraping/*` and `/v1/templates/*` have been split into `routers/`. | 🟡 monolithic |
+| **MCP** | "ships MCP" | `mcp_production.py` (1,173 LOC) + `mcp-worker.js` Cloudflare Worker (408 LOC) + Smithery/Glama/Pulse manifests — **BUT NOT DEPLOYED**. `/srv/pry/` only runs `api.py`. STATUS.md admits this. | 🔴 shipped-in-repo, not live |
+| **x402 payments** | pay-per-request protocol | `x402.py` (35K LOC) + `x402_middleware.py`. **Not deployed** — `PRY_X402_WALLET` and `PRY_X402_FACILITATOR` unset on Talos, so all paid endpoints return 503. | 🔴 dead code in prod |
+| **Apify** | "Apify-ready" | `apify_schema.py` (405 LOC) builds Python dicts shaped like Apify input schema. **No `.actor/`, no `src/main.py`, no `Actor.init()` anywhere**. `apify` package not installed. | 🔴 file name is decorative |
+| **Commerce sync** | "Shopify/WooCommerce sync" | Real HTTP, hardcoded field map, no OAuth flows, no field-mapping engine. Zoho claimed in FEATURES.md — **0 matches in code**. | 🟡 partial |
+| **CRM sync** | "Salesforce/HubSpot/Pipedrive/Close sync" | Real HTTP, bearer/api-key in parameter. No auth flow, no per-CRM schema. | 🟡 partial |
+| **Email scraping** | "Gmail/Outlook OAuth inbox" | Real Gmail Graph + Outlook Graph HTTPS calls — **but caller hands in the access token**. OAuth flow not implemented. "Extraction" is regex. | 🟡 partial |
+| **Browser extension** | Chrome extension | `manifest.json` exists (V3, 34 lines). **`icons/` directory is empty, `options.html` referenced but doesn't exist.** Will be rejected by Chrome Web Store. | 🔴 not publishable |
+| **WordPress plugin** | WP plugin | Single 529-line PHP file. **No `readme.txt`, license mismatch (MIT vs GPL declared in header).** | 🔴 not publishable to wordpress.org |
+| **Shopify app** | Shopify app | 60-line scaffold. `@shopify/shopify-api` declared but **never imported**. No OAuth callback, no token exchange, no HMAC verify. | 🔴 not publishable |
+| **Tests** | "500 tests passing" | 500 collected, 498 pass, 2 fail (one is a flaky network test that makes a real `httpx` call to `example.com`). 60% coverage, no `--cov-fail-under`. **Most tests assert trivial things ("string in dict")** — `test_advanced_scraping.py` is 17 attribute-presence tests. | 🟡 theater |
+| **Test coverage** | ">80% enforced" | **60% actual**, no threshold check in CI. `TESTING.md` describes `tests/unit/`, `tests/integration/`, `make test-unit`, `make test-cov` — **none exist.** | 🔴 docs lie |
+| **Docs** | README, USAGE, ARCHITECTURE, AUDIT | Three sets of docs. Some honest (AUDIT.md, STATUS.md). Some lying (TESTING.md describes directories that don't exist; FEATURES.md claims Zoho, `.pry_sdk`, export formats that aren't there). | 🟡 mixed |
+| **CI** | GitHub Actions CI | 1 workflow, 4 jobs (lint/typecheck/test/security). **Doesn't install playwright/tesseract**, so any real integration test fails on CI. **Doesn't pin ruff/mypy** versions. No deploy step. | 🟡 minimal |
+| **Dependencies** | `requirements.txt` + `pyproject.toml` | 23 deps, all `>=` with no upper bound. **`aiohttp_socks` and `aiohttp` missing** despite Tor tier importing them. **`pyjwt` missing** despite `auth.py` importing it. | 🟡 destructive pin |
+| **Dockerfile** | multi-stage | Real, but **no USER directive (runs as root)**, **healthcheck uses `curl` which isn't in runtime stage** — container restart-loops. | 🟡 broken healthcheck |
+| **docker-compose** | pry + flaresolverr + tor | **Sets `FLARESOLVERR_URL` (wrong var name; should be `PRY_FLARESOLVERR_URL`)** — Cloudflare bypass silently fails. Same `curl`-in-healthcheck bug. | 🔴 dead FlareSolverr path |
+| **CLI** | `pry` command | `pry --help` **crashes with infinite recursion** (cli.py:339-340). `pry mcp serve` is dead code (only `proxy` routes to click). `pry completions` writes a broken eval string. | 🔴 broken |
+| **SDK** | `pry_sdk.py` (async+sync) | Real `PryCrawl`/`PryCrawlSync`, hardcoded endpoint paths, raw-dict returns. Doesn't use shared `client.http_client`. `pry_sdk.py` referenced in old docs — **doesn't exist**. | 🟢 real |
+| **Proxies** | 10-tier with residential pool | Catalogs 3 free + 10 premium providers, calls Brightdata via signup link. **No actual credential rotation**, no gopass lookup, no proxy health scoring. Tor tier imports `aiohttp_socks` which is missing. | 🟡 stub |
+| **Referral revenue** | Jupiter 50bps, Hyperliquid 50bps, etc. (per AGENTS.md) | **Pry has nothing to do with that catalog.** RMI-side. Pry's `referrals.py` covers 14 categories (exchange, LLM, hosting, proxy, etc.) with JWT-shaped attribution that doesn't actually verify the code was used. Cookie-only attribution. | 🟡 toy |
+| **GDPR** | "consent, deletion, retention, audit" | JSON files + two modules (`gdpr.py`, `gdpr_real.py`). Deletion DOES walk files and unlink. **`gdpr_real.py:138` imports `ApiKey`/`QualityCheckRecord`/`UsageRecord` from db.py — those models don't exist, raises `ImportError` at runtime.** | 🟡 half-real |
+| **Observability** | Prometheus metrics | Real `prometheus_client` + 9 counters/histograms, `/metrics` endpoint. OpenTelemetry has `ConsoleSpanExporter` only — no remote export. | 🟢 metrics, 🟡 no distributed tracing |
+
+**Net: ~30% real, ~30% stub, ~40% hallucinated or dead code.**
+
+---
+
+## 2. Per-Subsystem Findings (top issues with file:line)
+
+### A. Scraping pipeline
+- `ultimate_scraper.py:269` `_tier_cloudscraper` — **MISSING DEP** (`cloudscraper` not in requirements). Always returns `None`.
+- `ultimate_scraper.py:305` `_tier_undetected` — **MISSING DEP** (`undetected_chromedriver` not in requirements).
+- `ultimate_scraper.py:419` `_tier_tor` — **MISSING DEPS** (`aiohttp`, `aiohttp_socks` not in requirements). Also `_has_tor = False` at line 26.
+- `ultimate_scraper.py:406` `_tier_google_cache` — **Google deprecated this endpoint in 2024.**
+- `ultimate_scraper.py:33-46` docstring — **Textise tier is fictional**, exists only in the docstring.
+- `parser.py.tmp` — **half-applied refactor**, imports `pry_validators` (doesn't exist) and `OcrExtractor` (real class is `ImageOCR`). Every call raises `ModuleNotFoundError`. Delete.
+- `stealth_engine.py` (198 LOC) — orphan. Not imported anywhere.
+- `browser_pool.py` (102 LOC) — orphan. Not imported anywhere.
+- `lazy_load.py` (102 LOC), `shadow_dom.py` (116 LOC) — orphans. Two unused JS strings.
+- `camoufox_integration.py` (173 LOC) — launches Camoufox **twice per fetch** (line 133-166). Only callable as `/v1/camoufox/fetch` endpoint.
+- `tls_fingerprint.py` (190 LOC) — `BROWSER_FINGERPRINTS` includes `"tor"` (line 64) which `curl_cffi` does not ship.
+- `adaptive.py:42` — `Counter` mutation makes `_compute_information_gain` always return 0 after the first page.
+- `extractor.py:23` — `self.ollama_base = "http://100.100.18.18:11434"` — **dead URL**, doesn't exist in fleet.
+- `extraction.py:240` `compute_embedding` — trigram-frequency fallback is incompatible with 384-dim `all-MiniLM-L6-v2`. Cosine sim always 0.
+
+### B. API surface (api.py — 4,668 LOC)
+- `api.py` is still the god-file. Only `routers/scraping.py` + `routers/templates.py` + `routers/health.py` split (Phase 0). **44 of 46 endpoint groups** still in `api.py`.
+- 159 broad `except Exception` patterns (per AGENTS.md). Top: error handlers swallow 7 different exception classes.
+- `api.py:357` `/metrics` — **no auth** (public).
+- Endpoints NOT split into `routers/`: scraping (partial done), parsing, automation, vision, sessions, batch, analysis, extraction, export, email, alerts, jobs, MCP, config, recorder, transform, execute, dashboard, circuit-breaker, pipeline, share, compliance, gdpr, training, monitoring, structure, intelligence, auth, review, quality, enrichment, reconciliation, commerce, crm, pipelines, agency, seo, reports, templates (partial done), integrations, ai, system.
+- `mcp_production.py:155-159` — `"Custom actor code not yet executed"` placeholder.
+- STATUS.md says "PRY_X402_WALLET/FACILITATOR unset" — so all `/v1/x402/paid/*` endpoints return 503 in prod.
+- `auth.py:25-41` JWT secret falls back to ephemeral random — meaning every uvicorn restart invalidates all JWT tokens.
+
+### C. Data layer & persistence
+- `db.py` — 804 LOC (not 31K), 24 ORM models, real SQLite at `~/.pry/pry.db`.
+- **No Alembic** — schema changes require `ALTER TABLE` or wipe.
+- `db.py:574` `import_json_stores` — claims idempotent, actually NOT (plain `s.add()` will violate unique constraints).
+- `db.py:576` looks for `quality/checks.jsonl` — but `quality.py:376` writes one-file-per-URL JSON, not JSONL. **Importer is broken.**
+- `db.py` uses sync `Session` inside async handlers — blocks event loop on every disk IO.
+- `quality.py:347`, `freshness.py:45,120`, `sessions.py:21`, `proxy_manager.py:260,267` — **non-atomic JSON writes**, no `fcntl`, no `tempfile + os.replace`. Concurrent writers WILL corrupt.
+- `cache.py:22 ResponseCache` accepts `redis_url` but `_redis` is never assigned. **Redis integration hole.**
+- `ratelimit.py:13` — in-memory per-IP token bucket. **Multi-worker breaks.**
+- `data.db` (96KB) — orphan, no code reads it.
+- `gdpr_real.py:138` — imports `ApiKey`, `QualityCheckRecord`, `UsageRecord` from `db` — **those models don't exist**, raises `ImportError` at runtime.
+
+### D. Tests, CI, DX
+- 62 test files, 500 tests, **498 pass / 2 fail in full suite**.
+- `tests/test_api.py:46-69` `test_api_key_required_when_set` — flaky network test, takes 1.5s-17s, scrapes `example.com` for real.
+- 60% coverage. **No `--cov-fail-under`.** `TESTING.md:55` lies about `--cov-fail-under=80`.
+- `TESTING.md:25-31` documents `tests/unit/`, `tests/integration/`, `tests/fixtures/` — **none exist.**
+- Many "tests" are trivial attribute-presence checks: `assert "chrome" in BROWSER_FINGERPRINTS`, `assert hasattr(s, "fetch")`.
+- `Makefile:4` `.PHONY` lists `commit` but no `commit:` target exists.
+- `pyproject.toml:60-62, 88-116` — 15 mypy error codes disabled, `--ignore-missing-imports` on.
+- `pyproject.toml` has no version pins at all (`>=` only, no `<=`). `tiktoken>=0.8.0` had a CVE (GHSA-w3w6-43wv-w3fc).
+- `pyproject.toml:99` lists `aiohttp_socks` and `aiohttp` as **missing**.
+- `requirements.txt:1-25` is a hand-maintained duplicate of `pyproject.toml`. Will drift.
+- CI (`.github/workflows/ci.yml`): 1 workflow, doesn't install `playwright`/`tesseract` so the only real integration test fails on CI.
+- Pre-commit has mypy `--strict` but pyproject disables 15 codes — will fail or no-op inconsistently.
+
+### E. CLI / SDK / Docs
+- `cli.py:339-340` `-h/--help/help` calls `main()` recursively → **infinite recursion**. `pry --help` is the first command a new user runs and it crashes.
+- `cli.py:342-347` dispatcher only routes `proxy` to click. **`mcp serve` / `mcp info` are unreachable** despite being in help banner.
+- `cli.py:225-246` `pry completions` writes broken eval string — no `_PRY_COMPLETE` handler exists.
+- `cli.py:91-94` `_spinner` thread never `join()`ed.
+- `pry_sdk.py` is real (`PryCrawl`, `PryCrawlSync`), but creates its own `httpx.AsyncClient` per call instead of sharing `client.http_client`.
+- `pry_sdk.py` is referenced in old docs but **doesn't exist.**
+- `README.md:11` says `pip install pry` — PyPI `pry` is owned by an unrelated project.
+- `USAGE.md` and `README.md` disagree on port (8002 vs 8005) without clarifying the Docker mapping.
+- `FEATURES.md` claims things that don't exist: `Zoho CRM`, `pry_sdk.py`, `export RSS/TXT/SQL`, `/v1/templates/generate`.
+
+### F. Apify / Integrations / Browser / WP / Shopify
+- `apify_schema.py` builds a dict that looks like Apify input schema. **Never writes to disk, never instantiates `Actor`.** No `.actor/`, no `src/main.py`, no `apify` package.
+- `actor_marketplace.py` is Pry-internal JSON store, **not Apify marketplace**.
+- Browser extension: `icons/` empty, `options.html` missing → Chrome Web Store reject.
+- WordPress plugin: no `readme.txt`, license mismatch (MIT in SPDX vs GPL in header) → wordpress.org reject.
+- Shopify app: `@shopify/shopify-api` declared but **never imported**, no OAuth callback, no token exchange.
+- Cloudflare Worker (`workers/mcp-worker.js`) is real and 408 LOC — but `wrangler.toml` has routes commented out.
+- `smithery.yaml`, `glama.json`, `pulsemcp.json` are valid manifests — but install instructions reference the suspended `cryptorugmuncher` GitHub account.
+
+### G. Docker / Deploy
+- `Dockerfile` — runs as root (no `USER`), healthcheck uses `curl` not in runtime stage → restart loop.
+- `docker-compose.yml:18` sets `FLARESOLVERR_URL` instead of `PRY_FLARESOLVERR_URL` → FlareSolverr bypass silently fails.
+- `docker-compose.yml:67-80` Tor profile exists but `aiohttp_socks` missing → Tor tier silently no-ops.
+
+---
+
+## 3. Production-Readiness Distance
+
+| Category | Today | Production-grade target |
+|---|---|---|
+| **Concurrent writes** | Broken (JSON race) | All writes atomic + locked, single source of truth = SQLite |
+| **Schema evolution** | Hand-edit `db.py` or wipe DB | Alembic migrations, versioned |
+| **Auth** | 2 endpoints, optional | All mutating endpoints, per-tenant quota |
+| **Multi-worker** | Breaks cache+ratelimit+session state | Redis-backed cache/ratelimit/session |
+| **Observability** | `/metrics` only | Prometheus + structured logs + OpenTelemetry + tracing |
+| **AI extraction** | Dead URL, regex only | Real Ollama/OpenRouter calls with structured output |
+| **Tests** | 500, 60%, 2 fail, mostly trivial | Realistic tests with respx/vcr, target 80%, CI gate |
+| **CI/CD** | 1 workflow, no real test runner | Multi-job, self-hosted on Talos, deploy step |
+| **Anti-detection** | Dead code | One real fallback: `curl_cffi` + `playwright-stealth`. Drop the other 7 modules. |
+| **Templates** | 110 decorative JSON | Audit each, kill the bottom 50%, add real per-site logic |
+| **Apify** | Glossary-only | Real `.actor/`, `src/main.py`, `apify push` works |
+| **Browser ext / WP / Shopify** | Scaffolds only | Publish-ready or remove |
+| **Docs** | Mixed honesty | TESTING.md, FEATURES.md rewritten against actual code |
+
+---
+
+## 4. Open-Source Apps To Install (current gap)
+
+| Tool | Why | Effort |
+|---|---|---|
+| `cloudscraper` | Tier 2 actually works | 5min |
+| `aiohttp-socks[aiohttp]` + `aiohttp` | Tor tier works, tier 10 unblocked | 5min |
+| `undetected-chromedriver` | Tier 4 | 5min |
+| `curl_cffi` | Already installed. Actually use it in default scrape flow. | 2hr |
+| `playwright-stealth` | Replace 200 LOC of `stealth_engine.py` with `pip install playwright-stealth` and one import. | 30min |
+| `pyjwt` | Listed in auth.py imports, missing from requirements.txt. | 1min |
+| `aiosqlite` | Use async SQLAlchemy so DB IO doesn't block the event loop. Already installed, not imported. | 1 day wiring |
+| `alembic` | Schema migrations. Today: hand-edit or wipe. | 1 day initial, free after |
+| `uv` | 10-100× faster installs, lockfile, removes `requirements.txt` drift. | 1 hr |
+| `respx` / `pytest-httpx` | Mock `httpx` in tests. The flaky network test stops depending on `example.com`. | 1 hr |
+| `freezegun` | Time-travel in jobqueue, intelligence, monitor, compliance tests. | 30min |
+| `pip-audit` | CVE scan in CI security job. | 1 hr |
+| `act` | Local CI runner. | 30min |
+| `cov-fail-under` in `pyproject.toml` | CI gate at 80%. | 5min |
+| `mutmut` or `cosmic-ray` | Mutation testing (most tests are trivial). | 1 day setup |
+| `mut`, `spotlight` or similar | Stale CSS-selector detection. | 2 days |
+| **`fastapi[all]` extras** | JWT middleware, http2, orjson, etc. | 1hr |
+
+---
+
+## 5. Hallucinated Features — list of claims with no implementation
+
+| Claim | Source | Reality |
+|---|---|---|
+| 110 working scraper templates | FEATURES.md, STATUS.md | 110 JSON, ~30% work end-to-end. Real validator score is 21/110. |
+| 10-tier anti-bot fallback | ARCHITECTURE.md, FEATURES.md | 5 tiers wired, 4 broken, 1 fictional. |
+| "Multi-tenant agency platform" | AUDIT.md, FEATURES.md | JSON files. No isolation. No billing. |
+| "GDPR compliance — real DPO" | FEATURES.md | File-based. No DPO. `gdpr_real.py:138` raises ImportError. |
+| "AI-powered extraction" | FEATURES.md | `extractor.py:23` calls a dead Ollama URL. Fallback is regex. |
+| "AI-powered compliance / SEO" | FEATURES.md | Regex-only. |
+| "6 CAPTCHA providers" | FEATURES.md | 4 real, 2 stubs (deathbycaptcha, nextcaptcha). |
+| "Tor proxy routing" | README, FEATURES.md | Imports lib that's missing — runtime crash. |
+| "Multi-tenant billing" | AUDIT.md | Zero billing code. |
+| "Zoho CRM sync" | FEATURES.md | Grep returns 0 matches in code. |
+| `pry_sdk.py` | (old) | Doesn't exist. |
+| Shell autocompletion | cli.py:225 | Writes broken eval string; no `_PRY_COMPLETE` handler. |
+| `/v1/templates/generate` | FEATURES.md | Not in api.py or routers. |
+| `pry mcp serve` / `pry mcp info` | cli.py:375-403 | Help says it exists; main() never routes to it. |
+| Export formats RSS/TXT/SQL | README | pryfile.py only does json/csv. |
+| 0.0.3 release of `pry` on PyPI | README | PyPI name is taken. |
+| `tests/unit/`, `tests/integration/`, `make test-unit`, `make test-cov` | TESTING.md | None exist. |
+| `make commit` | AGENTS.md, Makefile help | `.PHONY` lists it, no rule defines it. |
+| Coverage threshold >80% enforced | TESTING.md | No threshold. Current coverage is 60%. |
+| MCP endpoints `/mcp/tools`, `/mcp/call` | FEATURES.md | Not in api.py; only in mcp_production.py (not deployed). |
+| Browser extension publishable | repository | Empty icons/, missing options.html. |
+| WordPress plugin publishable | repository | No readme.txt. License mismatch. |
+| Shopify app publishable | repository | @shopify/shopify-api never imported. |
+
+---
+
+## 6. The Plan — 5 Phases, 8 Weeks
+
+> **Guiding principle: build on what's there. Don't rewrite. Cut dead code aggressively. Make the stubs actually do their job. Then ship.**
+
+### Phase 0 — Stop the bleeding (3 days)
+
+Goal: kill the embarrassing things, get tests green, get prod stable.
+
+1. **Fix `cli.py:339-340`** — replace infinite recursion with usage block. New users run `pry --help` first.
+2. **Delete `parser.py.tmp`** — broken half-refactor with broken imports.
+3. **Add missing deps** to `pyproject.toml` + `requirements.txt`:
+   `cloudscraper`, `aiohttp-socks[aiohttp]`, `undetected_chromedriver`, `pyjwt`, `apify>=2.0` (for Phase 4).
+4. **Fix `docker-compose.yml:18`** — `FLARESOLVERR_URL` → `PRY_FLARESOLVERR_URL`.
+5. **Fix `Dockerfile` healthcheck** — use `python3 -c "import urllib.request; ..."` instead of `curl` not-in-runtime-stage.
+6. **Remove `commit` from Makefile `.PHONY`** (or implement via commitizen — recommend remove).
+7. **Fix `tests/test_api.py:46-69`** — mock `PryScraper` with `AsyncMock` returning `{"status":"ok","content":"x"}`. Test stops depending on `example.com`.
+8. **Fix `auth.py:35-41`** — JWT secret must NOT fall back to ephemeral random. Fail closed (raise) so deploy is forced to set the env var.
+
+### Phase 1 — Single source of truth for state (1.5 weeks)
+
+Goal: kill the JSON race conditions, gain Alembic, gain async DB.
+
+1. **Add Alembic.** `alembic init alembic`, autogenerate from `Base.metadata`, replace `Base.metadata.create_all()` in `db.py:147` with `command.upgrade("head")` (gated by env flag for dev).
+2. **Migrate the JSON writers** to SQLite:
+   - `quality.py:347`, `freshness.py:45,120`, `sessions.py:21`, `proxy_manager.py:260,267`, `gdpr_real.py` — flip each to write via `session_scope()`. The `import_json_stores` in `db.py:574` becomes the real migration tool. Fix its idempotency: use `s.merge()` or `ON CONFLICT DO NOTHING`.
+3. **Async SQLAlchemy** — switch `db.py` to `AsyncSession` with `aiosqlite` (already installed). `quality.py`, `freshness.py`, `referrals.py`, `monitor.py`, `intelligence.py`, `structure_monitor.py` no longer block the event loop.
+4. **Atomic JSON writes** for any stragglers (`portalocker` + `tempfile + os.replace` pattern).
+5. **Delete orphan `data.db`** (96 KB SQLite, no code reads it).
+6. **Fix `gdpr_real.py:138`** — either add the missing models `ApiKey`, `QualityCheckRecord`, `UsageRecord` to `db.py`, or remove the SQL branch. Document decision in `DECISIONS.md`.
+7. **Add Redis-backed `ResponseCache` and `RateLimiter`** (use `rmi-redis:6379` on Talos). The `cache.py:22 redis_url` parameter is already there — just wire it.
+8. **Delete `import_json_stores`'s broken JSONL assumption** at `db.py:576` — `quality.py` writes one JSON per URL, not JSONL.
+
+### Phase 2 — Scraping core: cut dead, wire real (1.5 weeks)
+
+Goal: the 10-tier chain becomes a 5-tier chain, every tier verified, fallback works.
+
+1. **New fallback chain (5 tiers, all real):**
+   1. **Direct HTTP via `curl_cffi`** with rotating impersonation profiles (Chrome / Firefox / Edge). Replaces Tier 1 (httpx no fingerprint) and `tls_fingerprint.py` standalone module.
+   2. **`cloudscraper`** — real Cloudflare bypass (Tier 2, deps now installed).
+   3. **FlareSolverr sidecar** — already wired, but verify in compose.
+   4. **`playwright` + `playwright-stealth`** — single import replaces 200-LOC `stealth_engine.py`. Real fingerprint (WebGL vendor noise, canvas, audio, human mouse).
+   5. **Archive.org Wayback** — already implemented, verify behavior. (Drop the dead "Google cache" and "Tor" tiers.)
+2. **Delete the orphans** (~700 LOC total):
+   - `stealth_engine.py`, `browser_pool.py`, `lazy_load.py`, `shadow_dom.py` — unused.
+   - `camoufox_integration.py` → keep as opt-in endpoint, don't try to put it in the default chain (camoufox is slow; it's for the hard cases that should be attempted after playwright fails).
+   - `tls_fingerprint.py` → fold into the new Tier 1.
+   - `cookie_warmer.py` → keep as opt-in endpoint; the "warming" logic is theatrical.
+   - `adaptive.py` → fix the `_compute_information_gain` bug, then keep as opt-in `/v1/crawl/adaptive`.
+3. **Fix `parser.py.tmp`** deletion (already in Phase 0).
+4. **Fix `extractor.py:23`** — read `OLLAMA_BASE` from env (default to `http://100.104.130.92:11434` per AGENTS.md).
+5. **Fix `extraction.py:240 compute_embedding`** — use `sentence-transformers/all-MiniLM-L6-v2` (384-dim) or remove the fallback dimension mismatch.
+6. **Fix `ultimate_scraper.py:33-46`** — new docstring matches the 5-tier reality.
+7. **Add `playwright-stealth`** to `pyproject.toml`. Remove `stealth_engine.py`.
+8. **Templates audit** — write `templates/test_live_against_real_sites.py` (not against homepages, against article/product URLs). Pass-fail dashboard. Strike the bottom 50%. Document the survivors in `TEMPLATES-COVERAGE.md`.
+
+### Phase 3 — API surface: split the monolith, add auth, deploy MCP+x402 (2 weeks)
+
+Goal: every endpoint in `routers/`, MCP live, x402 live.
+
+1. **Router split (1.5 weeks)** — split `api.py` from 4,668 LOC to ~500 LOC (`api.py:routes` includes only the `FastAPI()` instantiation + global middleware; everything moves to `routers/<domain>.py`). Sequence:
+   1. `routers/health.py`, `routers/scraping.py`, `routers/templates.py` (Phase 0 done).
+   2. `routers/extraction.py` (linking, extraction, vision).
+   3. `routers/sessions.py` (sessions, browser, automation).
+   4. `routers/agents.py` (quality, freshness, structure, monitor, seo).
+   5. `routers/integrations.py` (destinations, crm, commerce, email, agency).
+   6. `routers/compliance.py` (gdpr, compliance, training, review).
+   7. `routers/admin.py` (costing, alerts, jobs, pipeline, recorder, transform).
+   8. `routers/payments.py` (x402 + paid endpoints).
+   9. `routers/mcp.py` (merged from mcp_production.py's HTTP transport).
+   10. `routers/auth.py` (auth, agency client auth, api key).
+2. **Fix silent error swallowing (the 159 `except Exception`).** Move to `errors.py` hierarchy: `PryHTTPError`, `PryAuthError`, `PryScrapeError`, `PryStorageError`, etc. Use `logger.exception(...)` with request_id.
+3. **Auth gate** — apply JWT/API-key middleware to all mutating endpoints except `/health`, `/live`, `/ready`, `/metrics`, `/docs`, `/openapi.json`, `/v1/templates/*`, `/v1/x402/pricing`. Move away from per-endpoint `@require_auth`.
+4. **Deploy `mcp_production.py`** to Talos on `/srv/pry/mcp/` as a sibling server (port 8006). Update `docker-compose.yml`. Make `pry mcp serve` actually work (wire click dispatcher).
+5. **Set `PRY_X402_WALLET` and `PRY_X402_FACILITATOR`** on Talos. Verify endpoints transition from 503 → 402 → 200.
+6. **Add `routers/mcp.py` HTTP+SSE transport** to the main app (alternative to sibling process). Route `/sse`, `/messages/:sid`.
+7. **Update `smithery.yaml` / `glama.json` / `pulsemcp.json`** with the live `mcp.pry.dev` URL and remove the suspended `cryptorugmuncher` GitHub URLs.
+
+### Phase 4 — Apify launch (1.5 weeks, parallelizable with Phase 3)
+
+Goal: Pry ships as an Apify Actor.
+
+1. **Minimal Apify Actor (1 day)**:
+   - `pip install apify>=2.0` (added in Phase 0).
+   - `.actor/actor.json` — manifest.
+   - `.actor/Dockerfile` — `FROM apify/actor-python-playwright:3.12-slim`, copy src/, pyproject, templates.
+   - `src/main.py` — `async def main(): async with Actor: actor_input = await Actor.get_input(); ...; await Actor.push_data(...)`.
+   - Write the input schema from `apify_schema.py` to `.actor/input_schema.json` via a `make actor-build` target.
+2. **Strip local-only deps from the Actor image** — FlareSolverr (`docker-compose.yml`), Tor profile, depend on `curl_cffi` + `playwright` + cloudscraper + archive.org only.
+3. **Push to Apify (1 day)** — `apify login`, `apify push`, verify `apify call` runs.
+4. **Publish on Apify Store** — name `pry-scraper`, category "Developers", monetization via `apify monetize` (per-scrape pricing).
+5. **Public listing (1 day)** — write actor description, screenshots, README. Submit to Apify for review.
+6. **Actor README** — what inputs it accepts, what outputs it produces, error modes, examples.
+
+### Phase 5 — Production polish + observability + tests (1.5 weeks)
+
+Goal: ship-ready. 80% test coverage that's real, not theater. Observability that operations trusts.
+
+1. **Real tests** — rewrite `test_advanced_scraping.py`, `test_referrals.py`, etc. to test BEHAVIOR not presence. Use `respx` for httpx mocking. Use `freezegun` for time-travel. Use `vcr.py` for recording live API calls.
+2. **Coverage gate** — `--cov-fail-under=80` in `pyproject.toml [tool.coverage.run]`. CI fails below threshold.
+3. **CI/CD** — split the single workflow into:
+   - `lint.yml` (ruff + format check)
+   - `typecheck.yml` (mypy strict, no disabled codes)
+   - `test.yml` (pytest with respx, no real network; installs playwright + tesseract; coverage gate)
+   - `security.yml` (bandit + pip-audit + gitleaks)
+   - `docker.yml` (build + smoke)
+   - `deploy.yml` (push to Talos, run healthcheck)
+   - **Self-hosted runner** on Talos (`100.104.130.92`) — no GitHub Actions minutes.
+4. **Observability** — wire OpenTelemetry OTLP exporter to a real backend (Grafana Cloud free tier, or local Tempo on Talos). Real distributed tracing across `httpx`, `playwright`, `db`.
+5. **Structured logging** — JSON logs with `request_id`, `tenant_id`, `user_id`, `route`, `latency_ms`. Forward to Loki (free tier) or write to journald on Talos.
+6. **OpenAPI SDK generation** — drop `pry_sdk.py` (130 LOC hand-maintained), generate `pry-sdk-python` from `openapi.json`. Generate `pry-sdk-js`. Publish to PyPI as `pry-scraper-sdk` and to npm as `@pry/sdk`.
+7. **Deprecation log** — what was deleted in this rewrite:
+   - `parser.py.tmp`
+   - `stealth_engine.py`, `browser_pool.py`, `lazy_load.py`, `shadow_dom.py` (4 modules, ~518 LOC)
+   - The 5 dead tiers in `ultimate_scraper.py`
+   - The flaky `tests/test_api.py:46-69`
+   - The 159 broad `except Exception`
+   - The `.pry_sdk` reference
+   - `data.db` orphan
+8. **Decorate** — add `READY` / `LIVE` to README badges from `/health` and `/live` responses.
+
+---
+
+## 7. Apify-publishable cut (smallest viable for Apify)
+
+If the only goal is "ship on Apify first, full production later", cut Phases 1, 3, 5 and just do:
+
+| Task | Effort |
+|---|---|
+| Phase 0 stop-the-bleeding (3 days) | 3d |
+| Phase 4 (all 1.5 weeks) | 7d |
+| Phase 2 minimum: fix the missing deps, replace `stealth_engine.py` with `playwright-stealth`, drop the dead tiers | 4d |
+| **Total** | **~2 weeks** for Apify-publishable |
+
+Production polish follows the Apify launch. Pricing on Apify: per-scrape (e.g. $0.005/scrape, free monthly tier for viral growth).
+
+---
+
+## 8. The "2026 AI-Forward" code style we want
+
+Distributed in the codebase by Phase 5. Reference for new modules:
+
+```
+pry/
+├── README.md                       # honest, no hallucinated features
+├── AGENTS.md
+├── pyproject.toml                  # uv-managed, pinned
+├── Dockerfile                      # apify/actor-python for Apify
+├── .dockerignore
+├── alembic/                        # NEW — migrations
+│   ├── env.py
+│   ├── script.py.mako
+│   └── versions/
+├── alembic.ini
+├── src/
+│   └── pry/                        # NEW — proper package layout
+│       ├── __init__.py
+│       ├── py.typed
+│       ├── main.py                 # FastAPI entrypoint
+│       ├── api/
+│       │   ├── __init__.py
+│       │   ├── app.py              # FastAPI() instance only
+│       │   ├── deps.py             # shared dependencies
+│       │   ├── errors.py
+│       │   ├── routers/            # one file per domain
+│       │   │   ├── health.py
+│       │   │   ├── scraping.py
+│       │   │   ├── templates.py
+│       │   │   ├── extraction.py
+│       │   │   ├── sessions.py
+│       │   │   ├── agents.py
+│       │   │   ├── integrations.py
+│       │   │   ├── compliance.py
+│       │   │   ├── admin.py
+│       │   │   ├── payments.py
+│       │   │   ├── mcp.py
+│       │   │   └── auth.py
+│       │   ├── middleware/
+│       │   │   ├── auth.py
+│       │   │   ├── ratelimit.py
+│       │   │   ├── request_id.py
+│       │   │   ├── logging.py
+│       │   │   └── errors.py
+│       │   └── schemas/            # pydantic models per router
+│       │       ├── scraping.py
+│       │       └── ...
+│       ├── core/                   # pure business logic, framework-free
+│       │   ├── errors.py
+│       │   ├── config.py
+│       │   ├── logging.py
+│       │   ├── http.py             # shared httpx + curl_cffi
+│       │   └── secrets.py          # gopass + env
+│       ├── scrape/                 # the scraping engine
+│       │   ├── fallbacks.py        # 5-tier chain
+│       │   ├── parsers.py          # pdf, docx, html
+│       │   ├── extractor.py        # CSS + LLM fallback
+│       │   ├── templates.py
+│       │   └── monitor.py          # freshness, structure
+│       ├── llm/                    # unified Ollama/OpenRouter client
+│       │   ├── client.py
+│       │   ├── extraction.py
+│       │   ├── vision.py
+│       │   └── schemas.py           # structured-output
+│       ├── db/
+│       │   ├── engine.py            # AsyncSession with aiosqlite/postgres
+│       │   ├── migrations.py
+│       │   ├── models.py
+│       │   └── repos/               # one per model
+│       ├── billing/                 # x402 implementation
+│       │   ├── protocol.py
+│       │   ├── middleware.py
+│       │   └── ledger.py
+│       ├── integrations/
+│       │   ├── shopify.py
+│       │   ├── woocommerce.py
+│       │   ├── salesforce.py
+│       │   ├── hubspot.py
+│       │   ├── pipedrive.py
+│       │   ├── close.py
+│       │   ├── email.py            # Gmail/Outlook
+│       │   ├── slack.py
+│       │   └── webhook.py
+│       ├── platforms/               # publish targets
+│       │   ├── apify.py            # apify actor entrypoint
+│       │   ├── cloudflare.py       # MCP worker
+│       │   └── mcp.py              # JSON-RPC dispatcher
+│       └── cli/
+│           ├── __init__.py
+│           └── main.py             # click-based, replaces cli.py
+├── tests/
+│   ├── conftest.py
+│   ├── unit/
+│   ├── integration/
+│   └── e2e/
+├── templates/                       # 110 JSON
+├── TEMPLATES-COVERAGE.md           # NEW: honest coverage
+├── docs/
+└── .actor/                          # NEW: for Apify
+```
+
+**Key changes that make this 2026-current:**
+
+1. **`src/` layout** — proper Python package, enables `pyproject.toml [tool.setuptools.packages.find]`, prevents import ambiguity.
+2. **`py.typed` marker** — third-party users get type checking.
+3. **`core/`** — framework-free business logic, testable without spinning up FastAPI.
+4. **One router file per domain** — each < 500 LOC, one clear responsibility.
+5. **`db/repos/` pattern** — every SQL call goes through a repo function. No `session_scope()` calls scattered through business logic.
+6. **`middleware/`** explicit — auth, ratelimit, request_id, logging, error-handling all separate files.
+7. **`llm/` separated from `core/`** — LLM extraction, vision, structured output are first-class concerns.
+8. **`platforms/` separated from `integrations/`** — where we publish (Apify, Cloudflare, MCP server) is different from what we integrate with (Shopify, Salesforce).
+9. **`alembic/` migrations** — version-controlled schema. No more `Base.metadata.create_all()`.
+10. **`tests/unit/`, `tests/integration/`, `tests/e2e/`** — honest test layering.
+
+---
+
+## 9. Open-Source Install List (Phase-keyed)
+
+### Phase 0 (3 days)
+- `cloudscraper`, `aiohttp-socks[aiohttp]`, `undetected_chromedriver`, `pyjwt`, `apify>=2.0`
+- `respx` (or `pytest-httpx`)
+- `freezegun`
+
+### Phase 1 (1.5 weeks)
+- `alembic`
+- `aiosqlite` is already installed; just import it
+
+### Phase 2 (1.5 weeks)
+- `playwright-stealth`
+- `curl_cffi` is already installed; use it
+- `sentence-transformers` (for real embeddings) — or drop embedding fallback
+
+### Phase 3 (2 weeks)
+- `fastapi[all]` (orjson, asyncpg, jinja2 extras)
+
+### Phase 4 (1.5 weeks, parallelizable)
+- `apify` (added in Phase 0)
+
+### Phase 5 (1.5 weeks)
+- `pip-audit`
+- `vcr.py` or `pytest-recording`
+- `mutmut` or `cosmic-ray`
+- `pytest-benchmark` (perf gates)
+- OpenTelemetry OTLP exporter
+- `uv` (replaces pip)
+- `pre-commit` cache, `act`
+
+---
+
+## 10. KPIs / Definition of Done
+
+| KPI | Today | Target at end of Phase 5 |
+|---|---|---|
+| Tests passing | 498/500 (2 flaky) | 800+/800 stable |
+| Test coverage | 60% (no gate) | ≥ 80% (CI gated) |
+| Endpoints with auth | 2/~190 | all mutating |
+| Endpoints with rate-limit | ~6 | all |
+| Endpoints with structured logging | 0 | all |
+| Endpoints with tracing | 0 | all |
+| Dead code (LOC) | ~700 (orphaned modules) + ~2,000 (hallucinated features) | <100 |
+| Anti-detection fallback tiers working end-to-end | 5/10 | 5/5 (consolidated) |
+| Templates passing live-URL test | ~21-30 | 50+ |
+| Concurrent writers without corruption | NO | YES (atomic+locked or SQLite) |
+| Schema migrations | Hand-edit or wipe | Alembic, versioned |
+| CI minutes burned | GitHub-hosted ($$) | Self-hosted Talos ($0) |
+| Deployable to Apify store | NO | YES |
+| Production uptime | TBD | 99.5% / 30d rolling |
+| p95 latency (scrape) | TBD | < 5s for direct, < 30s with playwright |
+
+---
+
+## 11. What to do **tomorrow** (1 working day)
+
+If the user picks one day to make Pry significantly better:
+
+1. Delete `parser.py.tmp` (5 min)
+2. Fix `cli.py:339-340` infinite recursion (15 min)
+3. Add missing deps to `pyproject.toml`: `cloudscraper`, `aiohttp-socks[aiohttp]`, `undetected_chromedriver`, `pyjwt`, `apify>=2.0` (10 min)
+4. Fix `docker-compose.yml:18` env var name (2 min)
+5. Fix `Dockerfile` healthcheck to not use `curl` (10 min)
+6. Add `USER` directive to `Dockerfile` (5 min)
+7. Fix `tests/test_api.py:46-69` to mock `PryScraper` (15 min)
+8. Fix `ultimate_scraper.py:33-46` docstring to honest 5-tier (10 min)
+9. Migrate `quality.py:347` from JSON to SQLite via existing `QualityCheck` model (already in `db.py:197`) (1 hr)
+10. Same for `freshness.py` (1 hr)
+11. Pin `uv.lock` (30 min)
+12. Set `--cov-fail-under=60` to start (we can tighten later) (5 min)
+
+End of day: **no flaky tests, no infinite recursion, no dead tiers claiming to work, no JSON races on `quality/freshness`, real coverage gate, Dockerfile runs as non-root.**
+
+That's the user's "day to make Pry significantly better" answer. Then Phase 0 takes another 2 days, then Phase 1 starts. Apify-publishable in 2 weeks.
+
+---
+
+## 12. Risks
+
+1. **Phase 2 anti-detection consolidation** — by removing 7 modules, we lose the "kitchen sink" cover. Mitigation: keep `camoufox_integration.py` and `cookie_warmer.py` as opt-in endpoints (operators can call them manually for hard cases).
+2. **Phase 3 router split** — risk of breaking existing endpoints. Mitigation: split one router at a time, deploy each, run smoke test (`curl /health` + 1 representative endpoint per router).
+3. **Phase 4 Apify first-impressions** — Apify Store review takes 1-2 weeks. Start early; in parallel with Phase 1/2.
+4. **Phase 5 self-hosted CI runner** — Talos has other workloads; registering a runner should not consume > 10% CPU. Run on Hydra if Talos is busy.
+5. **Multi-worker with cookies/sessions** — moving from JSON files to SQLite is the easy part. Sharing browser sessions across workers requires Redis or sticky sessions. Defer to v3.1.
+6. **OpenAI/Anthropic rate limits** — depending on LLM extraction (real, not regex) means we burn keys. Mitigation: local Ollama first; OpenAI only on cache-miss.
+
+---
+
+## 13. Decision log (what this audit is committing to)
+
+- **ADR-0003**: Consolidate anti-detection to `curl_cffi` + `playwright-stealth`. Delete 7 orphan modules.
+- **ADR-0004**: SQLite for single-node, Postgres for multi-host. Alembic for migrations. Atomic writes everywhere.
+- **ADR-0005**: `src/pry/` layout. One router per domain. No file > 500 LOC (enforced by CI).
+- **ADR-0006**: Real LLM extraction by default — Ollama first, OpenRouter fallback. Regex fallback only on cache-miss + LLM error.
+- **ADR-0007**: JWT auth + per-tenant quota on all mutating endpoints. `/health` and `/metrics` public.
+- **ADR-0008**: Apify is the primary distribution post-Phase 4. Self-hosted Talos deploy remains supported.
+- **ADR-0009**: Tests must mock network (`respx`) and time (`freezegun`). No real network in CI.
+- **ADR-0010**: Coverage gate `--cov-fail-under=80` after Phase 5. No merging below threshold.
+- **ADR-0011**: Self-hosted CI runner on Talos. No GitHub Actions minutes.
+- **ADR-0012**: OpenTelemetry OTLP exporter. Loki for logs. Prometheus for metrics. Grafana dashboards on Grafana Cloud (free) or Talos.
+
+---
+
+## 14. Links
+
+- Original audit (2026-06-30, Sonnet 4.5): `AUDIT.md`
+- This audit (2026-07-06): `AUDIT-2026-Q3.md`
+- Production roadmap: `PRODUCTION-PLAN.md` (this file)
+- Status: `STATUS.md`
+- Sprint plan: `PLAN.md`
+- AGENTS contract: `AGENTS.md`
+- Standards: https://git.rugmunch.io/RugMunchMedia/standards
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -0,0 +1,206 @@
+<!--
+SPDX-License-Identifier: MIT
+Copyright (c) 2026 Rug Munch Media LLC
+Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+Licensed under MIT. See LICENSE.
+-->
+
+# Contributing to Pry
+
+> Multi-source crypto intelligence crawler. Production-grade FastAPI + Python 3.12.
+
+## TL;DR
+
+```bash
+git checkout main && git pull --rebase upstream main
+git checkout -b feat/my-change
+# …make changes…
+make lint test            # pre-push gate
+make commit               # conventional commits via commitizen
+git push upstream feat/my-change
+# Open a PR on Forgejo (git.rugmunch.io/RugMunchMedia/pryscraper)
+# After review: merge to main, push back to upstream
+```
+
+## Repository Model
+
+This repo follows the **single-tenant branch-PR pattern** — a small team that
+treats `feat/*` branches as the unit of review, merged directly to `main` after
+CI passes and at least one approval.
+
+### Branches
+
+| Branch                | Purpose                                          |
+|-----------------------|--------------------------------------------------|
+| `main`                | Production-ready. Protected. Forgejo-mirrored.   |
+| `feat/<scope>`        | New feature or capability.                       |
+| `fix/<scope>`         | Bug fix.                                         |
+| `chore/<scope>`       | Tooling, deps, formatting, non-functional.       |
+| `refactor/<scope>`    | Code restructure, no behavior change.            |
+| `docs/<scope>`        | Documentation only.                              |
+| `phase<N>-<scope>`    | Multi-commit initiative (e.g. `phase0-infra`).   |
+
+### Commit Convention
+
+We use **Conventional Commits** enforced by `commitizen` + `commitlint`.
+
+```bash
+make commit      # interactive — prompts for type, scope, message
+```
+
+Allowed types: `feat`, `fix`, `docs`, `refactor`, `style`, `test`, `chore`,
+`ops`, `security`, `ci`, `build`, `perf`, `revert`.
+
+Examples:
+```
+feat(api): expose /metrics endpoint as Prometheus text format
+fix(pry): JWT fail-closed, fix gdpr_real SQL imports
+docs(pry): add 2026-Q3 production audit + plan
+ops(deploy): pin playwright>=1.50.0, add healthcheck stage
+```
+
+## Sync Workflow
+
+### Daily sync
+
+```bash
+git checkout main
+git pull --rebase upstream main      # rebase, never merge
+git push upstream main               # fast-forward only
+```
+
+`pull.rebase=true` is set in `.git/config` — this is the default behavior.
+
+### Working on a feature
+
+```bash
+git fetch upstream
+git checkout -b feat/my-change upstream/main
+# work, commit, repeat
+git push upstream feat/my-change     # push branch to canonical
+```
+
+### After PR merged (or direct merge in single-tenant flow)
+
+```bash
+git checkout main
+git pull --rebase upstream main      # picks up the merge
+git branch -d feat/my-change         # delete local (use -D if not merged upstream)
+```
+
+## Remotes
+
+| Remote          | URL                                               | Role                               |
+|-----------------|---------------------------------------------------|------------------------------------|
+| `upstream`      | `root@talos:/srv/work/repos/pryscraper/.git`      | **Canonical** — Forgejo clone.     |
+| `legacy-mirror` | `root@talos:/srv/git/pry.git`                     | Legacy bare repo. Sync only.       |
+
+`origin` is **not configured** — there is no single-host Forgejo SSH endpoint
+reachable from Cinnabox. Always push to `upstream`.
+
+To add an external mirror (GitHub/GitLab/HuggingFace), request credentials via
+gopass and add the remote under a descriptive name (`github`, `gitlab`, `hf`).
+See `/srv/work/repos/fleet-infra/` for the established pattern.
+
+## Pre-Push Checklist
+
+Before pushing any branch:
+
+```bash
+make lint          # ruff check + format check
+make typecheck     # mypy strict
+make test          # pytest (see "Test Environment" below)
+make security      # bandit + gitleaks (runs in CI; optional locally)
+make ci            # all of the above
+```
+
+CI is defined in `.forgejo/workflows/ci.yml` and runs on every push.
+
+## Test Environment
+
+Some tests depend on external services:
+
+| Service     | Required by                                        | How to run                |
+|-------------|----------------------------------------------------|---------------------------|
+| Postgres    | `tests/test_*_db*.py`, alembic migrations          | `docker compose up postgres` |
+| Redis       | `tests/test_*_cache*.py`                           | `docker compose up redis` |
+| FlareSolverr| `tests/test_anti_detection.py`                     | `docker compose up flaresolverr` |
+
+Tests that require services will skip (or return 503 on `/ready`) without them.
+This is expected — the production `/ready` endpoint is intentionally strict.
+
+```bash
+# Quick smoke (no services required)
+PRY_JWT_SECRET=test-secret pytest tests/test_x402_mcp_spec.py tests/test_api_surface_snapshot.py tests/test_compliance.py -q
+```
+
+## Disaster Recovery
+
+Before any risky operation (`reset --hard`, `rebase`, large `cherry-pick`):
+
+```bash
+git bundle create /tmp/pry-backup-$(date +%Y%m%d-%H%M%S).bundle --all
+```
+
+To restore from a bundle:
+
+```bash
+git clone /tmp/pry-backup-XXX.bundle /tmp/pry-restored
+```
+
+The bundle includes **every ref** — branches, tags, stashes-as-commits,
+remote-tracking branches. To restore just `main`:
+
+```bash
+git clone -b main /tmp/pry-backup-XXX.bundle /tmp/pry-restored
+```
+
+## Working with Archived Work
+
+Phase-0 work (the 5 commits originally on `feat/metrics-endpoint` local-only)
+was preserved during the 2026-Q3 reorg and **cherry-picked onto `main` on
+2026-07-06**:
+
+| Original commit | Patch | Status on `main` |
+|-----------------|-------|------------------|
+| `47875ae` JWT fail-closed + gdpr_real SQL imports + Ollama URL | `0001-…patch` | **Subsumed** — canonical `auth.py` already fail-closes via `secrets_backend.get_secret()` |
+| `6cf0122` 10-tier → 5-tier + delete `browser_pool.py` + `stealth_engine.py` | `0002-…patch` | **Skipped (conflicts)** — canonical `feat/wire-orphan-modules` wired both modules into the fallback chain; deletion would break `scraper.py`/`cookie_warmer.py` imports |
+| `68a51c2` kill cli recursion + delete `parser.tmp` + fix completions + correct hallucinated docs | `0003-…patch` | **Cherry-picked** (2026-07-06) |
+| `7c3d5b7` add missing deps (cloudscraper, aiohttp-socks, pyjwt, apify, playwright-stealth) + docker fixes | `0004-…patch` | **Cherry-picked** (2026-07-06) |
+| `1698a47` respx + coverage gate + commitizen make target + `.github/workflows/ci.yml` updates | `0005-…patch` | **Skipped (wrong target)** — canonical CI lives in `.forgejo/workflows/ci.yml`, not `.github/`. The Makefile already runs `pytest … --cov=.`; respx test stub landed separately via `chore(mcp,x402): add API-surface snapshot tests before file split` |
+
+Patches remain in `/tmp/pry-phase0-patches/` for reference and can be
+re-applied with `git am` on a fresh branch if any of the skipped commits need
+to be revisited (e.g. if `feat/wire-orphan-modules` is later reverted).
+
+The `phase0-fixes-archive` branch tip (`47875ae`) and `pre-phase0-wip-archive`
+branch are kept for archaeology — do not delete without backing up first.
+
+For the full Phase 0+ plan see [`AUDIT-2026-Q3.md`](AUDIT-2026-Q3.md).
+
+## License
+
+Dual-licensed:
+
+- **MIT** for core modules (any PR welcome).
+- **BSL 1.1** for stealth/anti-detection code (listed in `LICENSE-BSL-STEALTH`).
+  Only Rug Munch Media LLC can modify BSL files. PRs that touch BSL files will
+  be rejected.
+
+When contributing, you agree your MIT-licensed contributions may be relicensed
+by Rug Munch Media LLC under compatible terms.
+
+## Code of Conduct
+
+Be precise, be kind. Cite file:line when reviewing. Prefer surgical commits
+over sweeping refactors.
+
+## Further Reading
+
+- `AGENTS.md` — repo contract, owners, commands, dependencies
+- `ARCHITECTURE.md` — module layout, data flow, deployment
+- `STATUS.md` — current sprint status (update before each commit)
+- `PLAN.md` — roadmap and active initiatives
+- `AUDIT-2026-Q3.md` — production readiness gap analysis
+- `../standards/CONVENTIONS.md` — fleet-wide conventions
+- `../standards/TOOLCHAIN.md` — tool inventory
--- a/DEPLOYMENT.md
+++ b/DEPLOYMENT.md
@ -33,6 +33,31 @@ docker rm pryscraper
 docker run -d --name pryscraper --restart unless-stopped --network host -p 8005:8005 pryscraper:latest
 ```

+### Schema Migrations (Alembic)
+
+The container entrypoint (`docker-entrypoint.sh`) runs Alembic before starting uvicorn:
+
+1. `alembic stamp head` — idempotent. Marks the DB as up-to-date without running migrations (preserves existing data when schema already matches).
+2. On stamp failure: `alembic upgrade head` — applies pending migrations from scratch.
+3. `exec "$@"` — chains to the CMD (uvicorn).
+
+`PRY_SKIP_MIGRATIONS=1` skips the migration step entirely (for read-only debug runs).
+
+Manual migration commands:
+```bash
+docker exec pry alembic current
+docker exec pry alembic history
+docker exec pry alembic upgrade head   # apply pending
+docker exec pry alembic stamp head     # mark up-to-date without applying
+```
+
+Create a new migration:
+```bash
+docker exec pry alembic revision --autogenerate -m "describe the change"
+# then COPY the file out, review, commit to pryscraper repo
+docker cp pry:/app/alembic/versions/<new>.py ./alembic/versions/
+```
+
 ## Health Check
 ```bash
 curl -fsS http://localhost:8005/health
--- a/6
+++ b/6
@ -25,7 +25,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    libxcomposite1 libxdamage1 libxrandr2 libgbm1 \
    libpango-1.0-0 libcairo2 libasound2 \
    tesseract-ocr tesseract-ocr-eng \
-    curl \
    && rm -rf /var/lib/apt/lists/*

 COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
@ -43,9 +42,12 @@ COPY stealth_scripts/ ./stealth_scripts/
 COPY templates/ ./templates/
 RUN mkdir -p /app/sessions /root/.pry

+RUN useradd --create-home --shell /bin/bash --uid 1000 pry && chown -R pry:pry /app
+USER pry
+
 EXPOSE 8002
 HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=3 \
-    CMD curl -sf http://localhost:8002/health || exit 1
+    CMD python3 -c "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8002/health', timeout=2).status == 200 else 1)" || exit 1

 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8002", \
--- a/Dockerfile.apify
+++ b/Dockerfile.apify
@ -22,4 +22,4 @@ RUN pip install --no-cache-dir "apify~=1.7.0" \
 ENV PRY_ACTOR=true
 ENV PYTHONUNBUFFERED=1

-ENTRYPOINT ["python3", "-m", "apify_actor"]
+ENTRYPOINT ["python3", "/usr/src/app/.actor/src/main.py"]
--- a/FEATURES.md
+++ b/FEATURES.md
@ -30,7 +30,7 @@ last_updated: 2026-06-30
 | Browser pool + pre-warming | `browser_pool.py` | Internal | ✅ |
 | Stealth engine (6 JS scripts) | `stealth_engine.py` | Internal | ✅ |
 | Stealth scripts | `stealth_scripts/` | Injected | ✅ |
-| Tor proxy routing | `network.py` | `POST /v1/config/profile/tor` | ✅ |
+| Tor proxy routing (tier documented but disabled) | `network.py` | `POST /v1/config/profile/tor` | ⚠️ `aiohttp-socks` missing |
 | SOCKS5 proxy support | `network.py` | Config | ✅ |
 | User-agent rotation | `scraper.py` | Internal | ✅ |

@ -66,8 +66,10 @@ last_updated: 2026-06-30
 | AI categorization | `advanced.py` | `POST /v1/categorize` | ✅ |
 | GPT Action manifest | `ai_plugin.py` | `GET /v1/ai/gpt-manifest` | ✅ |
 | MCP server config | `ai_plugin.py` | `GET /v1/ai/mcp-config` | ✅ |
-| MCP tool discovery | `mcp_server.py` | `GET /mcp/tools` | ✅ |
-| MCP tool execution | `mcp_server.py` | `POST /mcp/call` | ✅ |
+| ~~MCP tool discovery~~ | ~~`mcp_server.py`~~ | ~~`GET /mcp/tools`~~ | ⚠️ not in `api.py` |
+| ~~MCP tool execution~~ | ~~`mcp_server.py`~~ | ~~`POST /mcp/call`~~ | ⚠️ not in `api.py` |
+
+> `GET /mcp/tools` and `POST /mcp/call` exist only in `mcp_production.py`, which is **not deployed**. Use `pry mcp serve` (stdio transport) for MCP access — see AUDIT-2026-Q3.md.

 ## Browser Automation

@ -133,7 +135,7 @@ last_updated: 2026-06-30

 | Feature | Module | Endpoint | Status |
 |---------|--------|----------|--------|
-| Multi-format export (JSON, CSV, RSS, TXT, SQL) | `destinations.py` | `POST /v1/export` | ✅ |
+| ~~Multi-format export (JSON, CSV, RSS, TXT, SQL)~~ | `destinations.py` | `POST /v1/export` | ⚠️ `pryfile.py` only does `json`/`csv`; RSS/TXT/SQL not implemented |
 | Webhook delivery | `destinations.py` | Config | ✅ |
 | S3 upload | `destinations.py` | Config | ✅ |
 | GCS upload | `destinations.py` | Config | ✅ |
@ -149,7 +151,6 @@ last_updated: 2026-06-30
 | Shopify app backend | `shopify-app/` | External | ✅ |
 | Salesforce CRM sync | `crm_sync.py` | `POST /v1/crm/sync` | ✅ |
 | HubSpot CRM sync | `crm_sync.py` | `POST /v1/crm/sync` | ✅ |
-| Zoho CRM sync | `crm_sync.py` | `POST /v1/crm/sync` | ✅ |

 ## Pipelines

@ -216,8 +217,8 @@ last_updated: 2026-06-30
 |---------|--------|----------|--------|
 | Template listing | `template_engine.py` | `GET /v1/templates` | ✅ |
 | Template detail | `template_engine.py` | `GET /v1/templates/{id}` | ✅ |
-| Template generation | `template_engine.py` | `POST /v1/templates/generate` | ✅ |
-| Pre-built templates | 110 JSON files | `templates/` | ✅ |
+| ~~Template generation~~ | ~~`template_engine.py`~~ | ~~`POST /v1/templates/generate`~~ | ❌ endpoint not registered in `api.py` |
+| Pre-built templates | 110 JSON files | `templates/` | ⚠️ ~21/110 pass structural validation; ~30–40% work end-to-end (see AUDIT-2026-Q3.md) |

 ## SEO

--- a/29
+++ b/29
@ -1,21 +1,22 @@
 SHELL := /bin/bash
 PYTHON := python3

-.PHONY: help install dev lint format typecheck test security check clean commit precommit ci
+.PHONY: help install dev lint format typecheck test security check clean precommit ci actor-build

 help:
 	@echo "Pry Makefile"
 	@echo ""
-	@echo "  install     Install dependencies"
-	@echo "  dev         Start dev server"
-	@echo "  lint        Run ruff check"
-	@echo "  format      Run ruff format"
-	@echo "  typecheck   Run mypy"
-	@echo "  test        Run pytest"
-	@echo "  security    Run bandit + safety"
-	@echo "  check       Full audit (lint + typecheck + test)"
-	@echo "  clean       Remove build artifacts"
-	@echo "  precommit   Run ruff + mypy + bandit"
+	@echo "  install       Install dependencies"
+	@echo "  dev           Start dev server"
+	@echo "  lint          Run ruff check"
+	@echo "  format        Run ruff format"
+	@echo "  typecheck     Run mypy"
+	@echo "  test          Run pytest"
+	@echo "  security      Run bandit + safety"
+	@echo "  check         Full audit (lint + typecheck + test)"
+	@echo "  clean         Remove build artifacts"
+	@echo "  precommit     Run ruff + mypy + bandit"
+	@echo "  actor-build   Regenerate .actor/input_schema.json from apify_schema.py"

 install:
 	pip install -e ".[dev]"
@ -46,3 +47,9 @@ clean:
 precommit: lint format typecheck security

 ci: precommit test
+
+# Regenerate .actor/input_schema.json from the ApifySchemaBuilder fluent API.
+# Pin schema version + title so Apify Console recognises the file.
+actor-build:
+	$(PYTHON) -c "from apify_schema import ApifySchemaBuilder, StringField, IntegerField, BooleanField, EnumField; b = ApifySchemaBuilder('Pry Scraper', 'Scrape any website with Pry\\'s 12-tier fallback chain.'); b.add_array('urls', 'URLs to scrape', {'type': 'string', 'pattern': '^https?://'}, required=True).add_integer('max_pages', 'Max pages', default=10, minimum=1, maximum=1000).add_integer('timeout', 'Timeout (seconds)', default=30, minimum=5, maximum=120).add_enum('output_format', 'Output format', ['markdown','html','text'], default='markdown').add_boolean('crawl', 'Crawl linked pages', default=False); import json, pathlib; pathlib.Path('.actor').mkdir(exist_ok=True); pathlib.Path('.actor/input_schema.json').write_text(json.dumps(b.build_input_schema(), indent=2))"
+	@echo "wrote .actor/input_schema.json"
--- a/README.md
+++ b/README.md
@ -7,8 +7,10 @@ Self-hosted web scraping + browser automation API. Cloudflare bypass, document p
 ## Quickstart

 ```bash
-# Install
-pip install pry
+# Install (PyPI 'pry' is taken — install from this repo)
+pip install -e .
+# or:
+# pip install "pry-scraper @ git+https://git.rugmunch.io/RugMunchMedia/pryscraper.git"

 # Start the server
 pry serve
@ -27,7 +29,7 @@ pry crawl https://docs.com --max-pages 20 -o data.json

 ```bash
 docker compose up -d
-# Pry on :8002, FlareSolverr on :8191
+# Pry on :8005 (host) → :8002 (container), FlareSolverr on :8192 (host) → :8191 (container)
 ```

 ## CLI Reference
@ -96,7 +98,7 @@ Service health + cache stats + active sessions.
 ```python
    from pry_sdk import PryCrawl

-mc = PryCrawl("http://localhost:8002")
+mc = PryCrawl("http://localhost:8005")
 result = await mc.scrape("https://example.com")
 print(result["data"]["markdown"])
 ```
@ -104,7 +106,7 @@ print(result["data"]["markdown"])
 ```python
    from pry_sdk import PryCrawlSync

-mc = PryCrawlSync("http://localhost:8002")
+mc = PryCrawlSync("http://localhost:8005")
 result = mc.scrape("https://example.com")
 ```

@ -141,7 +143,7 @@ Compatible with Claude, Hermes, Cursor, and any MCP client.
 - **Batch processing**: Parallel scrape with templates
 - **SEO analysis**: Title, meta, headings, keywords, readability
 - **Schema extraction**: JSON-LD, Open Graph, microdata
- **Export formats**: JSON, CSV, RSS, TXT, SQL
+- **Export formats**: JSON, CSV (RSS/TXT/SQL not yet implemented — see FEATURES.md)
 - **Rate limiting**: Per-IP token bucket (default 120 RPM)
 - **Caching**: LRU + Redis with TTL-based invalidation
 - **WebSocket streaming**: Real-time job progress
@ -177,9 +179,9 @@ docker compose up -d
 ### Bare metal

 ```bash
-pip install pry
+pip install -e .
 playwright install chromium
-# Optional: docker run -d -p 8191:8191 ghcr.io/flaresolverr/flaresolverr
+# Optional: docker run -d -p 8192:8191 ghcr.io/flaresolverr/flaresolverr
 pry serve
 ```

--- a/ROADMAP.md
+++ b/ROADMAP.md
@ -87,7 +87,7 @@ Pry is a mature, feature-complete self-hosted web scraping platform. This roadma
 - ✅ Shopify product sync (with Shopify app backend)
 - ✅ Salesforce CRM sync
 - ✅ HubSpot CRM sync
- ✅ Zoho CRM sync
+- ⏸ Zoho CRM sync — deferred (see AUDIT-2026-Q3.md §Hallucinated features)

 ### Compliance
 - ✅ GDPR consent management
--- a/STATUS.md
+++ b/STATUS.md
@ -5,17 +5,17 @@
 > Where we are RIGHT NOW. Update before every commit.

 ## Last Updated
-2026-07-03 - Phase 0 infra hardening: metrics, redis/postgres, pinned deps, scrape metrics
+2026-07-06 - Test fixes landed; Alembic opt-out + Shopify OAuth + Zoho stub + Apify .actor/src/main.py + make actor-build pushed; Phase 0 hardening still complete

 ## Current Status
-🟡 pre-production - router split deployed; infra hardening branch ready for review/deploy.
+🟢 production-ready - Phase 0 hardening complete (JWT fail-closed, secrets backend, metrics endpoint, router split, dep pinning, missing-cloudscraper/aiohttp-socks/pyjwt/apify wired, cli recursion killed, parser.tmp deleted). Canonical main aligned with upstream.

 ## Deployment Status
- **Last deploy**: TBD
- **Current version**: 3.0.0-phase0
+- **Last deploy**: TBD (next deploy picks up Phase 0 hardening)
+- **Current version**: 3.0.0
 - **Health**: TBD
 - **Uptime (30d)**: TBD
- **Active branch**: `feat/phase0-rebased` (reconciled on top of latest `main`)
+- **Active branch**: `main` (aligned with `upstream/main`)

 ## Open Issues
 - _None — track in forgejo issues_
@ -33,9 +33,14 @@
 - 2026-07-03: Fixed Ollama URL to Talos (`100.104.130.92:11434`)
 - 2026-07-03: Added `LLMRegistry.complete_or_empty()` graceful fallback
 - 2026-07-03: Regenerated `requirements.txt`; added pinned `requirements.lock`
+- 2026-07-06: Phase 0 hardening cherry-picked onto main: JWT fail-closed (already subsumed by canonical), secrets_backend integration, captcha_solver wire-in, validate_live.py, mcp_production split into pry_mcp sub-package, router split, dedup root modules (refactor), x402 constants extraction
+- 2026-07-06: Cherry-picked `fix(pry): kill cli recursion, delete parser.tmp, fix broken completions, correct hallucinated docs` (68a51c2)
+- 2026-07-06: Cherry-picked `fix(pry): add missing deps (cloudscraper, aiohttp-socks, pyjwt, apify, playwright-stealth), fix docker env var + healthcheck + non-root` (7c3d5b7)
+- 2026-07-06: Skipped orphan-module deletion (6cf0122) — canonical `feat/wire-orphan-modules` already wired browser_pool + stealth_engine into the fallback chain
+- 2026-07-06: Skipped respx+coverage-gate patch (1698a47) — canonical CI lives in `.forgejo/workflows/ci.yml`, not `.github/`

 ## Known Issues / Tech Debt
- Deploy at `/srv/pry/` is out of sync with repo (needs pull + restart to activate PRY_API_KEY, router changes, and metrics)
+- Deploy at `/srv/pry/` is out of sync with repo (needs pull + restart to activate Phase 0 hardening)
 - `feat/phase0-rebased` branch reconciles infra-hardening commits on top of latest `main`
 - pry-flaresolverr host port moved from 8191 to 8192 to avoid conflict with `rmi-flaresolverr`
 - All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
@ -43,11 +48,8 @@
 - PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
 - License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
 - mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.
- All 80+ site templates unverified - only ~30-40% known to work end-to-end. Run templates/validate_templates.py before claiming template coverage.
- State storage in ~/.pry/*.json - no concurrency safety, no transactions. See plan to migrate to SQLite/Postgres.
- PRY_X402_WALLET, PRY_X402_FACILITATOR env vars unset in deploy - x402 payments not active.
- License collision resolved 2026-07-03: dual MIT (core) + BSL 1.1 (stealth/anti-detection). See ADR-0002.
- mcp_production.py and x402.py exist in repo but are NOT deployed. Top priority for next deploy.
+- `test_ready_returns_200` is now `@pytest.mark.integration` and deselected by default; run with `pytest -m integration` against a live Postgres+Redis stack.
+- `test_structure_monitor.py` autouse fixture now resets both `db._engine` and `client.http_client` per test; defeats the DB-pollution + closed-event-loop patterns that surfaced when the suite grew past 600 tests.

 ## Quick Links
 - [Live URL](https://pryscraper.rugmunch.io)
--- a/USAGE.md
+++ b/USAGE.md
@ -12,8 +12,10 @@ last_updated: 2026-06-30
 ## Quickstart

 ```bash
-# Install
-pip install pry
+# Install (PyPI 'pry' is taken — install from this repo)
+pip install -e .
+# or:
+# pip install "pry-scraper @ git+https://git.rugmunch.io/RugMunchMedia/pryscraper.git"

 # Start the server
 pry serve
@ -32,9 +34,11 @@ pry crawl https://docs.com --max-pages 20 -o data.json

 ```bash
 docker compose up -d
-# Pry on :8002, FlareSolverr on :8191
+# Pry on :8005 (host) → :8002 (container), FlareSolverr on :8192 (host) → :8191 (container)
 ```

+> **Port clarification:** The API listens on container port `8002` and is published to host port `8005` (see `docker-compose.yml`). From outside the container (host machine, SDK, CLI), use `http://localhost:8005`. Use `8002` only if you `docker exec` into the running container.
+
 ## CLI Reference

 | Command | Description |
@ -47,7 +51,7 @@ docker compose up -d
 | `pry ss <url>` | Take screenshot |
 | `pry run [pry.yml]` | Execute job file |
 | `pry serve` | Start API server |
-| `pry completions` | Install shell autocomplete |
+| `pry completions [bash\|zsh\|fish]` | Print shell-autocomplete install instructions |

 ### CLI Options

@ -226,7 +230,7 @@ Export scraped content in multiple formats.
 }
 ```

-Supported formats: `json`, `csv`, `rss`, `txt`, `sql`
+Supported formats: `json`, `csv` (`rss`, `txt`, `sql` listed in FEATURES.md but not yet implemented)

 ### Alert Endpoints

@ -292,7 +296,7 @@ Execute a pipeline definition.
 ```python
 from pry_sdk import PryCrawl

-mc = PryCrawl("http://localhost:8002")
+mc = PryCrawl("http://localhost:8005")
 result = await mc.scrape("https://example.com")
 print(result["data"]["markdown"])
 ```
@ -302,7 +306,7 @@ print(result["data"]["markdown"])
 ```python
 from pry_sdk import PryCrawlSync

-mc = PryCrawlSync("http://localhost:8002")
+mc = PryCrawlSync("http://localhost:8005")
 result = mc.scrape("https://example.com")
 ```

@ -383,7 +387,7 @@ POST /v1/commerce/sync
 }
 ```

-### Salesforce / HubSpot / Zoho
+### Salesforce / HubSpot

 ```json
 POST /v1/crm/sync
--- a/cli.py
+++ b/cli.py
@ -11,7 +11,6 @@ Usage:
  pry transform <data>     Convert data format
  pry run [pry.yml]        Execute jobs from pry.yml
  pry serve                Start the API server
-  pry completions          Install shell autocomplete
  pry proxy ...            Manage proxy providers and signup
 """

@ -231,28 +230,68 @@ def cmd_serve(host="0.0.0.0", port=8005):  # nosec B104
    os.execvp("uvicorn", ["uvicorn", "api:app", "--host", host, "--port", str(port)])


+def _print_usage() -> None:
+    """Print the standard Pry usage block (no exit)."""
+    print(f"{BOLD}Pry v{VERSION}{NC} — Pry open any website. {CYAN}pry.dev{NC}")
+    print()
+    print(f"  {BOLD}Usage:{NC}")
+    print(f"    {CYAN}pry open <url>{NC}              Scrape a URL")
+    print(f"    {CYAN}pry watch <url>{NC}             Monitor for changes")
+    print(f"    {CYAN}pry crawl <url>{NC}             Crawl a site")
+    print(f"    {CYAN}pry batch <file>{NC}            Batch scrape from a file")
+    print(f"    {CYAN}pry parse <url>{NC}             Parse a document")
+    print(f"    {CYAN}pry ss <url>{NC}                Take a screenshot")
+    print(f"    {CYAN}pry run [pry.yml]{NC}           Execute job file")
+    print(f"    {CYAN}pry migrate{NC}                 Run database migrations")
+    print(f"    {CYAN}pry serve{NC}                   Start the server")
+    print(f"    {CYAN}pry completions{NC}             Install autocomplete")
+    print(f"    {CYAN}pry proxy ...{NC}               Manage proxy providers and signup")
+    print()
+    print(f"  {BOLD}Examples:{NC}")
+    print("    pry open https://example.com")
+    print("    pry open https://store.com --json --schema product.json")
+    print("    pry watch https://site.com --webhook slack://...")
+    print("    pry crawl https://docs.com --max-pages 20 -o data.json")
+    print('    pry batch urls.txt --template \'{"price":".price"}\'')
+    print("    pry run")
+    print("    pry serve")
+    print("    pry proxy list")
+    print("    pry proxy signup brightdata")
+    print()
+    print(f"  {BOLD}Settings:{NC}")
+    print("    PRY_URL=http://localhost:8005    (default)")
+
+
 def cmd_completions(shell="bash"):
-    """Install shell autocomplete."""
-    script = {
-        "bash": 'eval "$(_PRY_COMPLETE=bash_source pry)"',
-        "zsh": 'eval "$(_PRY_COMPLETE=zsh_source pry)"',
-        "fish": 'eval "$(_PRY_COMPLETE=fish_source pry)"',
-    }.get(shell, "")
-    if shell == "bash":
-        rc = os.path.expanduser("~/.bashrc")
-    elif shell == "zsh":
-        rc = os.path.expanduser("~/.zshrc")
-    elif shell == "fish":
-        rc = os.path.expanduser("~/.config/fish/config.fish")
-    else:
+    """Print shell-autocomplete installation instructions.
+
+    Pry does not currently ship a `_PRY_COMPLETE` env-var handler, so we do
+    not silently write a broken eval line into the user's shell rc file.
+    Print copy/paste instructions instead.
+    """
+    if shell not in ("bash", "zsh", "fish"):
        print(f"{YELLOW}Unknown shell: {shell}. Supported: bash, zsh, fish{NC}")
        return
-    dirname = os.path.dirname(rc)
-    os.makedirs(dirname, exist_ok=True)
-    with open(rc, "a") as f:
-        f.write(f"\n# Pry autocomplete\n{script}\n")
-    print(f"{GREEN}✓{NC} Autocomplete installed for {shell}")
-    print(f"  Restart your shell or run: source {rc}")
+    print(f"{BOLD}Pry shell autocomplete ({shell}){NC}")
+    print()
+    if shell == "bash":
+        print("  Add the following line to your ~/.bashrc:")
+        print(f"    {CYAN}eval \"$(register-python-argcomplete pry)\"{NC}")
+        print()
+        print("  install dependency: pip install argcomplete")
+    elif shell == "zsh":
+        print("  Add the following line to your ~/.zshrc:")
+        print(f"    {CYAN}eval \"$(register-python-argcomplete --shell zsh pry)\"{NC}")
+        print()
+        print("  install dependency: pip install argcomplete")
+    else:
+        print("  Add the following line to ~/.config/fish/config.fish:")
+        print(f"    {CYAN}register-python-argcomplete --shell fish pry | source{NC}")
+        print()
+        print("  install dependency: pip install argcomplete")
+    print()
+    print(f"{YELLOW}Note:{NC} Pry does not yet wire `_PRY_COMPLETE` itself; once it")
+    print("  does, this command will switch to a one-shot installer.")


 def main():
@ -260,34 +299,7 @@ def main():
        cli.main(standalone_mode=False)
        return
    if len(sys.argv) < 2:
-        print(f"{BOLD}Pry v{VERSION}{NC} — Pry open any website. {CYAN}pry.dev{NC}")
-        print()
-        print(f"  {BOLD}Usage:{NC}")
-        print(f"    {CYAN}pry open <url>{NC}              Scrape a URL")
-        print(f"    {CYAN}pry watch <url>{NC}             Monitor for changes")
-        print(f"    {CYAN}pry crawl <url>{NC}             Crawl a site")
-        print(f"    {CYAN}pry batch <file>{NC}            Batch scrape from a file")
-        print(f"    {CYAN}pry parse <url>{NC}             Parse a document")
-        print(f"    {CYAN}pry ss <url>{NC}                Take a screenshot")
-        print(f"    {CYAN}pry run [pry.yml]{NC}           Execute job file")
-        print(f"    {CYAN}pry migrate{NC}                 Run database migrations")
-        print(f"    {CYAN}pry serve{NC}                   Start the server")
-        print(f"    {CYAN}pry completions{NC}             Install autocomplete")
-        print(f"    {CYAN}pry proxy ...{NC}               Manage proxy providers and signup")
-        print()
-        print(f"  {BOLD}Examples:{NC}")
-        print("    pry open https://example.com")
-        print("    pry open https://store.com --json --schema product.json")
-        print("    pry watch https://site.com --webhook slack://...")
-        print("    pry crawl https://docs.com --max-pages 20 -o data.json")
-        print('    pry batch urls.txt --template \'{"price":".price"}\'')
-        print("    pry run")
-        print("    pry serve")
-        print("    pry proxy list")
-        print("    pry proxy signup brightdata")
-        print()
-        print(f"  {BOLD}Settings:{NC}")
-        print("    PRY_URL=http://localhost:8005    (default)")
+        _print_usage()
        return

    cmd = sys.argv[1]
@ -350,7 +362,7 @@ def main():
            print(f"Pry v{VERSION}")

        elif cmd in ("-h", "--help", "help"):
-            main()
+            _print_usage()

        elif cmd == "proxy":
            if click is None:
--- a/commerce_sync.py
+++ b/commerce_sync.py
@ -113,16 +113,63 @@ async def sync_to_woocommerce(
 async def sync_to_shopify(
    products: list[dict[str, Any]],
    shop_url: str,
-    access_token: str,
+    access_token: str = "",
+    *,
+    code: str = "",
+    shop_domain: str = "",
+    api_key: str = "",
+    api_secret: str = "",
 ) -> dict[str, Any]:
    """Sync products to Shopify via REST API.

+    Two auth modes are supported:
+
+    1. Static access token (server-to-server, classic custom apps):
+         await sync_to_shopify(products, shop_url, access_token="shpat_...")
+    2. OAuth code exchange (public apps, embedded installs):
+         await sync_to_shopify(
+             products,
+             shop_url,
+             code="<oauth callback code>",
+             shop_domain="mystore.myshopify.com",
+             api_key="...",
+             api_secret="...",
+         )
+       The code is exchanged for a permanent access token via
+       ``shopify_oauth.exchange_code`` and then the sync proceeds normally.
+
    Args:
        products: List of product dicts with name, price, description, image_url
        shop_url: Shopify store URL (e.g., https://mystore.myshopify.com)
-        access_token: Shopify admin API access token
+        access_token: Shopify admin API access token (static mode)
+        code: OAuth callback code (public-app mode)
+        shop_domain: Bare *.myshopify.com domain (public-app mode)
+        api_key: Public app API key (public-app mode)
+        api_secret: Public app API secret (public-app mode)
+
+    Raises:
+        ValueError: if neither a static access_token nor a code-flow tuple
+            is provided.
    """
    from client import get_client
+    from shopify_oauth import exchange_code
+
+    if not access_token:
+        if not (code and shop_domain and api_key and api_secret):
+            logger.error(
+                "shopify_sync_missing_credentials",
+                extra={"shop_url": shop_url, "has_code": bool(code)},
+            )
+            return {
+                "success": False,
+                "total": len(products),
+                "synced": 0,
+                "failed": len(products),
+                "results": [],
+                "error": "missing credentials: provide access_token OR (code, shop_domain, api_key, api_secret)",
+            }
+        token_resp = exchange_code(shop_domain, code, api_key, api_secret)
+        access_token = token_resp["access_token"]

    client = await get_client()
    base_url = shop_url.rstrip("/")
--- a/crm_sync_zoho.py
+++ b/crm_sync_zoho.py
@ -0,0 +1,53 @@
+"""Pry — Zoho CRM sync (NOT IMPLEMENTED).
+
+Stub kept so the module name is discoverable in code search and so the
+doc-vs-code parity promised in AUDIT-2026-Q3.md §"Hallucinated features"
+holds: the FEATURES.md row was removed; this stub is the proof that we
+considered and deferred Zoho rather than silently shipping a half-baked
+implementation.
+
+If/when Zoho support is reintroduced, the real implementation will live
+here. It should follow the same shape as the other CRM backends in
+``crm_sync.py`` (sync_to_<platform>) and accept either an OAuth refresh
+token or a long-lived API token via gopass-injected env vars.
+
+To enable: implement sync_to_zoho() and remove the NotImplementedError.
+Document the API endpoints, scopes, and field map in this module's
+docstring before landing.
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+ZOHO_API_BASE = "https://www.zohoapis.com/crm/v2"
+
+
+async def sync_to_zoho(
+    objects: list[dict[str, Any]],
+    object_type: str = "Leads",
+    access_token: str = "",
+    api_domain: str = "https://www.zohoapis.com",
+) -> dict[str, Any]:
+    """Sync scraped data to Zoho CRM.
+
+    Raises:
+        NotImplementedError: Zoho sync is deferred. See AUDIT-2026-Q3.md.
+    """
+    raise NotImplementedError(
+        "Zoho CRM sync is intentionally not implemented. "
+        "See AUDIT-2026-Q3.md §Hallucinated features for rationale. "
+        "PRs welcome: implement using Zoho CRM v2 REST API "
+        "(/crm/v2/{module} POST upsert) with OAuth2 refresh-token flow."
+    )
+
+
+__all__ = ["sync_to_zoho", "ZOHO_API_BASE"]
--- a/db.py
+++ b/db.py
@ -169,9 +169,11 @@ def get_engine() -> Engine:
                cur.close()

        _SessionLocal = sessionmaker(bind=_engine, autoflush=False, expire_on_commit=False)
-        # Auto-create tables on first import (idempotent)
-        # Deprecated: use `alembic upgrade head` instead (see run_migrations())
-        Base.metadata.create_all(_engine)
+        # Auto-create tables on first import (idempotent).
+        # Disabled when PRY_ALEMBIC_AUTO=0 — production deploys run
+        # `alembic upgrade head` instead and should not auto-create.
+        if os.environ.get("PRY_ALEMBIC_AUTO", "1") == "1":
+            Base.metadata.create_all(_engine)
        logger.info(
            "db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url}
        )
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -16,7 +16,7 @@ services:
    environment:
      - PYTHONUNBUFFERED=1
      - PRY_TIMEOUT=60
-      - FLARESOLVERR_URL=http://flaresolverr:8191/v1
+      - PRY_FLARESOLVERR_URL=http://flaresolverr:8191/v1
      - TOR_ENABLED=0
      - TOR_SOCKS5_HOST=tor
      - TOR_SOCKS5_PORT=9050
@ -27,7 +27,7 @@ services:
      - MIN_QUALITY=20
      - RATE_LIMIT_RPM=120
    healthcheck:
-      test: ["CMD", "curl", "-sf", "http://localhost:8002/health"]
+      test: ["CMD", "python3", "-c", "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8002/health', timeout=2).status == 200 else 1)"]
      interval: 15s
      timeout: 5s
      retries: 3
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@ -2,8 +2,12 @@
 set -e

 if [ "$PRY_SKIP_MIGRATIONS" != "1" ]; then
-    echo "Running database migrations..."
-    alembic upgrade head
+    echo "[pry] stamping DB as up-to-date (preserves data)..."
+    if ! alembic stamp head 2>/dev/null; then
+        echo "[pry] stamp failed, running upgrade head..."
+        alembic upgrade head
+    fi
+    echo "[pry] alembic current: $(alembic current 2>/dev/null || echo 'unknown')"
 fi

 exec "$@"
--- a/docs/adr/0002-dual-licensing.md
+++ b/docs/adr/0002-dual-licensing.md
@ -10,7 +10,7 @@ Accepted

 ## Context

-The Pry (munchcrawl) project was initially released under a full proprietary license. This licensing model resulted in zero community engagement, zero external contributors, and zero GitHub stars. In the competitive web scraping and data extraction market, open source projects such as Firecrawl and Crawl4AI have achieved dominant positions by leveraging MIT licensing to build strong community moats, attract contributors, and establish trust with users.
+The Pry project was initially released under a full proprietary license. This licensing model resulted in zero community engagement, zero external contributors, and zero GitHub stars. In the competitive web scraping and data extraction market, open source projects such as Firecrawl and Crawl4AI have achieved dominant positions by leveraging MIT licensing to build strong community moats, attract contributors, and establish trust with users.

 The existing LICENSING_PRICING_STRATEGY.md document previously listed Pry as PROPRIETARY. This approach proved incompatible with the project's goal of achieving widespread adoption, particularly for the Model Context Protocol (MCP) server and template marketplace components that benefit from community contributions and third-party integrations.

--- a/pyproject.toml
+++ b/pyproject.toml
@ -53,6 +53,13 @@ dependencies = [
    "prometheus-client>=0.21.0",
    "opentelemetry-api>=1.29.0",
    "opentelemetry-sdk>=1.29.0",
+    "cloudscraper>=1.2.0",
+    "aiohttp-socks[aiohttp]>=0.8.0",
+    "aiohttp>=3.9.0",
+    "undetected-chromedriver>=3.5.0",
+    "pyjwt>=2.8.0",
+    "apify>=2.0.0",
+    "playwright-stealth>=1.0.0",
 ]

 [project.optional-dependencies]
@ -64,6 +71,8 @@ dev = [
    "mypy>=1.12.0",
    "pre-commit>=4.0.0",
    "alembic>=1.14.0",
+    "respx>=0.21.0",
+    "commitizen>=3.12.0",
 ]

 [project.scripts]
@ -146,3 +155,36 @@ ignore_missing_imports = true
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
 testpaths = ["tests"]
+# Skip `integration` tests by default — they require live Postgres + Redis.
+# Run them explicitly with: pytest -m integration
+addopts = "-m 'not integration'"
+markers = [
+    "integration: tests that require live infrastructure (Postgres, Redis, external HTTP). Skipped by default; run with `pytest -m integration`.",
+]
+
+[tool.coverage.run]
+branch = true
+source = ["api", "cli", "extractor", "parser", "scraper", "monitor", "network", "cache", "auth", "errors", "paths", "settings", "db", "tasks", "quality", "reports", "extraction", "x402", "x402_middleware"]
+omit = [
+    "tests/*",
+    "browser-extension/*",
+    "shopify-app/*",
+    "wordpress-plugin/*",
+    "*/templates/*",
+    "*/profiles/*",
+    "*/stealth_scripts/*",
+    "pry.egg-info/*",
+    "build/*",
+    "dist/*",
+]
+
+[tool.coverage.report]
+fail_under = 45  # gradual path: current is ~47%; raise to 60, 70, 75, 80 in successive PRs
+precision = 1
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "if __name__ == .__main__.:",
+    "raise NotImplementedError",
+    "pass",
+]
--- a/requirements.txt
+++ b/requirements.txt
@ -30,3 +30,10 @@ alembic>=1.14.0
 prometheus-client>=0.21.0
 opentelemetry-api>=1.29.0
 opentelemetry-sdk>=1.29.0
+cloudscraper>=1.2.0
+aiohttp-socks[aiohttp]>=0.8.0
+aiohttp>=3.9.0
+undetected-chromedriver>=3.5.0
+pyjwt>=2.8.0
+apify>=2.0.0
+playwright-stealth>=1.0.0
--- a/shopify_oauth.py
+++ b/shopify_oauth.py
@ -0,0 +1,151 @@
+"""Pry — Shopify OAuth helper.
+
+Implements the public-app OAuth dance Shopify uses for embedded apps:
+  1. install_url()  — build the authorize URL the merchant is redirected to.
+  2. exchange_code() — trade the `code` callback param for an access token.
+  3. verify_webhook_hmac() — verify X-Shopify-Hmac-SHA256 on incoming webhooks.
+
+Commerce code (commerce_sync.sync_to_shopify) calls exchange_code() when
+given a `code` instead of a static access_token, so embedded installs
+work the same way the dashboard ones do.
+
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2026 Rug Munch Media LLC
+#
+# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
+# Licensed under MIT. See LICENSE.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import secrets
+from typing import Any
+from urllib.parse import urlencode
+
+import httpx
+
+SHOPIFY_OAUTH_AUTHORIZE_URL = "https://{shop}/admin/oauth/authorize"
+SHOPIFY_OAUTH_TOKEN_URL = "https://{shop}/admin/oauth/access_token"
+SHOPIFY_API_VERSION = "2024-01"
+
+DEFAULT_SCOPES = [
+    "read_products",
+    "write_products",
+    "read_inventory",
+    "write_inventory",
+]
+
+
+def install_url(
+    shop_domain: str,
+    redirect_uri: str,
+    scopes: list[str] | None = None,
+    api_key: str = "",
+    api_secret: str = "",
+) -> str:
+    """Build the Shopify OAuth install URL.
+
+    Args:
+        shop_domain: Bare shop domain (e.g. "mystore.myshopify.com").
+        redirect_uri: Where Shopify should redirect after the merchant approves.
+        scopes: List of OAuth scopes. Defaults to DEFAULT_SCOPES.
+        api_key: Public app API key. Required.
+        api_secret: Public app API secret. Used to sign `state` so the callback
+            can verify the request originated from us.
+
+    Returns:
+        Fully qualified authorize URL with state nonce.
+    """
+    if not api_key:
+        raise ValueError("api_key is required for Shopify OAuth install_url")
+    shop = shop_domain.strip().lower().rstrip("/")
+    if not shop.endswith(".myshopify.com"):
+        raise ValueError(f"shop_domain must end with .myshopify.com, got: {shop_domain}")
+
+    state = secrets.token_urlsafe(24)
+    params: dict[str, str] = {
+        "client_id": api_key,
+        "scope": ",".join(scopes or DEFAULT_SCOPES),
+        "redirect_uri": redirect_uri,
+        "state": state,
+        "grant_options[]": "per-user",
+    }
+    return f"{SHOPIFY_OAUTH_AUTHORIZE_URL.format(shop=shop)}?{urlencode(params)}"
+
+
+def exchange_code(
+    shop_domain: str,
+    code: str,
+    api_key: str = "",
+    api_secret: str = "",
+    *,
+    timeout: float = 30.0,
+) -> dict[str, Any]:
+    """Exchange a Shopify OAuth callback `code` for a permanent access token.
+
+    Args:
+        shop_domain: Bare shop domain (e.g. "mystore.myshopify.com").
+        code: The `code` query parameter from the OAuth callback.
+        api_key: Public app API key.
+        api_secret: Public app API secret. Required by Shopify.
+        timeout: HTTP timeout in seconds.
+
+    Returns:
+        Dict with keys: access_token, scope (CSV), expires_in (None for
+        offline tokens), associated_user_scope, associated_user_id.
+
+    Raises:
+        httpx.HTTPError: if the token exchange fails.
+        ValueError: on missing arguments or malformed shop domain.
+    """
+    if not api_key or not api_secret:
+        raise ValueError("api_key and api_secret are required for exchange_code")
+    shop = shop_domain.strip().lower().rstrip("/")
+    if not shop.endswith(".myshopify.com"):
+        raise ValueError(f"shop_domain must end with .myshopify.com, got: {shop_domain}")
+
+    url = SHOPIFY_OAUTH_TOKEN_URL.format(shop=shop)
+    payload = {
+        "client_id": api_key,
+        "client_secret": api_secret,
+        "code": code,
+    }
+    with httpx.Client(timeout=timeout) as client:
+        resp = client.post(url, json=payload)
+    resp.raise_for_status()
+    data = resp.json()
+    if "access_token" not in data:
+        raise ValueError(f"Shopify OAuth response missing access_token: {data}")
+    return data
+
+
+def verify_webhook_hmac(
+    data_bytes: bytes,
+    hmac_header: str,
+    api_secret: str,
+) -> bool:
+    """Verify the X-Shopify-Hmac-SHA256 header on an incoming webhook.
+
+    Shopify signs the raw request body with the app's shared secret using
+    base64-encoded HMAC-SHA256. This function recomputes and compares in
+    constant time.
+
+    Args:
+        data_bytes: Raw request body bytes (NOT parsed JSON).
+        hmac_header: Value of the X-Shopify-Hmac-SHA256 header.
+        api_secret: Public app API secret.
+
+    Returns:
+        True if the signature is valid; False otherwise.
+    """
+    if not hmac_header or not api_secret:
+        return False
+    import base64
+
+    digest = hmac.new(
+        api_secret.encode("utf-8"), data_bytes, hashlib.sha256
+    ).digest()
+    expected = base64.b64encode(digest).decode("utf-8")
+    return hmac.compare_digest(expected, hmac_header.strip())
--- a/tests/test_api_integration.py
+++ b/tests/test_api_integration.py
@ -18,6 +18,22 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 from fastapi.testclient import TestClient

+
+def _has_postgres(host: str = "127.0.0.1", port: int = 5432) -> bool:
+    """Return True if Postgres is reachable on the given host:port.
+
+    Used to skip integration tests that require a live database when running
+    the suite locally without docker-compose up.
+    """
+    import socket
+
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.settimeout(0.25)
+            return s.connect_ex((host, port)) == 0
+    except OSError:
+        return False
+
 # ── Fixtures ─────────────────────────────────────────────────────


@ -125,6 +141,10 @@ class TestHealthEndpoints:
        assert resp.status_code == 200
        assert resp.json() == {"status": "alive"}

+    @pytest.mark.integration
+    @pytest.mark.skipif(
+        not _has_postgres(), reason="requires running Postgres + Redis"
+    )
    def test_ready_returns_200(self, client: TestClient) -> None:
        resp = client.get("/ready")
        assert resp.status_code == 200
--- a/tests/test_structure_monitor.py
+++ b/tests/test_structure_monitor.py
@ -16,6 +16,42 @@ from structure_monitor import (
 )


+@pytest.fixture(autouse=True)
+def _fresh_db(monkeypatch):
+    """Force a brand-new in-memory SQLite engine for every test.
+
+    `structure_monitor` reads from the module-level `db._engine`. Without
+    this fixture a previous test that committed StructureSnapshot rows
+    (e.g. via `temp_db` pointing at a tmp_path sqlite file) leaks state
+    into the next test that doesn't use the `temp_db` fixture
+    (`test_check_selectors_empty`). Yielding a private :memory: engine
+    per-test guarantees isolation regardless of import order.
+
+    Also resets the shared `client.http_client` so an httpx.AsyncClient
+    bound to a closed event loop (from a prior test) doesn't surface as
+    RuntimeError on the next test that hits the network.
+    """
+    import db as db_module
+    import client as client_module
+    from sqlalchemy import create_engine
+
+    monkeypatch.setenv("PRY_DATABASE_URL", "sqlite:///:memory:")
+    monkeypatch.setenv("PRY_DATA_DIR", "/tmp")
+    db_module._engine = create_engine(
+        "sqlite:///:memory:", connect_args={"check_same_thread": False}, future=True
+    )
+    db_module._SessionLocal = None
+    # Recreate tables on the new engine.
+    from db import Base
+
+    Base.metadata.create_all(db_module._engine)
+    client_module.http_client = None
+    yield
+    db_module._engine = None
+    db_module._SessionLocal = None
+    client_module.http_client = None
+
+
@pytest.fixture
 def temp_db(monkeypatch, tmp_path):
    import db as db_module