diff --git a/FEATURES.md b/FEATURES.md index 959090c..ce9b6f3 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -30,7 +30,7 @@ last_updated: 2026-06-30 | Browser pool + pre-warming | `browser_pool.py` | Internal | ✅ | | Stealth engine (6 JS scripts) | `stealth_engine.py` | Internal | ✅ | | Stealth scripts | `stealth_scripts/` | Injected | ✅ | -| Tor proxy routing | `network.py` | `POST /v1/config/profile/tor` | ✅ | +| Tor proxy routing (tier documented but disabled) | `network.py` | `POST /v1/config/profile/tor` | ⚠️ `aiohttp-socks` missing | | SOCKS5 proxy support | `network.py` | Config | ✅ | | User-agent rotation | `scraper.py` | Internal | ✅ | @@ -66,8 +66,10 @@ last_updated: 2026-06-30 | AI categorization | `advanced.py` | `POST /v1/categorize` | ✅ | | GPT Action manifest | `ai_plugin.py` | `GET /v1/ai/gpt-manifest` | ✅ | | MCP server config | `ai_plugin.py` | `GET /v1/ai/mcp-config` | ✅ | -| MCP tool discovery | `mcp_server.py` | `GET /mcp/tools` | ✅ | -| MCP tool execution | `mcp_server.py` | `POST /mcp/call` | ✅ | +| ~~MCP tool discovery~~ | ~~`mcp_server.py`~~ | ~~`GET /mcp/tools`~~ | ⚠️ not in `api.py` | +| ~~MCP tool execution~~ | ~~`mcp_server.py`~~ | ~~`POST /mcp/call`~~ | ⚠️ not in `api.py` | + +> `GET /mcp/tools` and `POST /mcp/call` exist only in `mcp_production.py`, which is **not deployed**. Use `pry mcp serve` (stdio transport) for MCP access — see AUDIT-2026-Q3.md. ## Browser Automation @@ -133,7 +135,7 @@ last_updated: 2026-06-30 | Feature | Module | Endpoint | Status | |---------|--------|----------|--------| -| Multi-format export (JSON, CSV, RSS, TXT, SQL) | `destinations.py` | `POST /v1/export` | ✅ | +| ~~Multi-format export (JSON, CSV, RSS, TXT, SQL)~~ | `destinations.py` | `POST /v1/export` | ⚠️ `pryfile.py` only does `json`/`csv`; RSS/TXT/SQL not implemented | | Webhook delivery | `destinations.py` | Config | ✅ | | S3 upload | `destinations.py` | Config | ✅ | | GCS upload | `destinations.py` | Config | ✅ | @@ -149,7 +151,7 @@ last_updated: 2026-06-30 | Shopify app backend | `shopify-app/` | External | ✅ | | Salesforce CRM sync | `crm_sync.py` | `POST /v1/crm/sync` | ✅ | | HubSpot CRM sync | `crm_sync.py` | `POST /v1/crm/sync` | ✅ | -| Zoho CRM sync | `crm_sync.py` | `POST /v1/crm/sync` | ✅ | +| ~~Zoho CRM sync~~ | ~~`crm_sync.py`~~ | ~~`POST /v1/crm/sync`~~ | ❌ not implemented (grep: 0 matches in code) | ## Pipelines @@ -216,8 +218,8 @@ last_updated: 2026-06-30 |---------|--------|----------|--------| | Template listing | `template_engine.py` | `GET /v1/templates` | ✅ | | Template detail | `template_engine.py` | `GET /v1/templates/{id}` | ✅ | -| Template generation | `template_engine.py` | `POST /v1/templates/generate` | ✅ | -| Pre-built templates | 110 JSON files | `templates/` | ✅ | +| ~~Template generation~~ | ~~`template_engine.py`~~ | ~~`POST /v1/templates/generate`~~ | ❌ endpoint not registered in `api.py` | +| Pre-built templates | 110 JSON files | `templates/` | ⚠️ ~21/110 pass structural validation; ~30–40% work end-to-end (see AUDIT-2026-Q3.md) | ## SEO diff --git a/Makefile b/Makefile index 6704333..9a485fa 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ SHELL := /bin/bash PYTHON := python3 -.PHONY: help install dev lint format typecheck test security check clean commit precommit ci +.PHONY: help install dev lint format typecheck test security check clean precommit ci help: @echo "Pry Makefile" diff --git a/README.md b/README.md index 62efc01..d3fc7ce 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,10 @@ Self-hosted web scraping + browser automation API. Cloudflare bypass, document p ## Quickstart ```bash -# Install -pip install pry +# Install (PyPI 'pry' is taken — install from this repo) +pip install -e . +# or: +# pip install "pry-scraper @ git+https://git.rugmunch.io/RugMunchMedia/pryscraper.git" # Start the server pry serve @@ -27,7 +29,7 @@ pry crawl https://docs.com --max-pages 20 -o data.json ```bash docker compose up -d -# Pry on :8002, FlareSolverr on :8191 +# Pry on :8005 (host) → :8002 (container), FlareSolverr on :8192 (host) → :8191 (container) ``` ## CLI Reference @@ -96,7 +98,7 @@ Service health + cache stats + active sessions. ```python from pry_sdk import PryCrawl -mc = PryCrawl("http://localhost:8002") +mc = PryCrawl("http://localhost:8005") result = await mc.scrape("https://example.com") print(result["data"]["markdown"]) ``` @@ -104,7 +106,7 @@ print(result["data"]["markdown"]) ```python from pry_sdk import PryCrawlSync -mc = PryCrawlSync("http://localhost:8002") +mc = PryCrawlSync("http://localhost:8005") result = mc.scrape("https://example.com") ``` @@ -141,7 +143,7 @@ Compatible with Claude, Hermes, Cursor, and any MCP client. - **Batch processing**: Parallel scrape with templates - **SEO analysis**: Title, meta, headings, keywords, readability - **Schema extraction**: JSON-LD, Open Graph, microdata -- **Export formats**: JSON, CSV, RSS, TXT, SQL +- **Export formats**: JSON, CSV (RSS/TXT/SQL not yet implemented — see FEATURES.md) - **Rate limiting**: Per-IP token bucket (default 120 RPM) - **Caching**: LRU + Redis with TTL-based invalidation - **WebSocket streaming**: Real-time job progress @@ -177,9 +179,9 @@ docker compose up -d ### Bare metal ```bash -pip install pry +pip install -e . playwright install chromium -# Optional: docker run -d -p 8191:8191 ghcr.io/flaresolverr/flaresolverr +# Optional: docker run -d -p 8192:8191 ghcr.io/flaresolverr/flaresolverr pry serve ``` diff --git a/USAGE.md b/USAGE.md index 89347be..e760b1e 100644 --- a/USAGE.md +++ b/USAGE.md @@ -12,8 +12,10 @@ last_updated: 2026-06-30 ## Quickstart ```bash -# Install -pip install pry +# Install (PyPI 'pry' is taken — install from this repo) +pip install -e . +# or: +# pip install "pry-scraper @ git+https://git.rugmunch.io/RugMunchMedia/pryscraper.git" # Start the server pry serve @@ -32,9 +34,11 @@ pry crawl https://docs.com --max-pages 20 -o data.json ```bash docker compose up -d -# Pry on :8002, FlareSolverr on :8191 +# Pry on :8005 (host) → :8002 (container), FlareSolverr on :8192 (host) → :8191 (container) ``` +> **Port clarification:** The API listens on container port `8002` and is published to host port `8005` (see `docker-compose.yml`). From outside the container (host machine, SDK, CLI), use `http://localhost:8005`. Use `8002` only if you `docker exec` into the running container. + ## CLI Reference | Command | Description | @@ -47,7 +51,7 @@ docker compose up -d | `pry ss ` | Take screenshot | | `pry run [pry.yml]` | Execute job file | | `pry serve` | Start API server | -| `pry completions` | Install shell autocomplete | +| `pry completions [bash\|zsh\|fish]` | Print shell-autocomplete install instructions | ### CLI Options @@ -226,7 +230,7 @@ Export scraped content in multiple formats. } ``` -Supported formats: `json`, `csv`, `rss`, `txt`, `sql` +Supported formats: `json`, `csv` (`rss`, `txt`, `sql` listed in FEATURES.md but not yet implemented) ### Alert Endpoints @@ -292,7 +296,7 @@ Execute a pipeline definition. ```python from pry_sdk import PryCrawl -mc = PryCrawl("http://localhost:8002") +mc = PryCrawl("http://localhost:8005") result = await mc.scrape("https://example.com") print(result["data"]["markdown"]) ``` @@ -302,7 +306,7 @@ print(result["data"]["markdown"]) ```python from pry_sdk import PryCrawlSync -mc = PryCrawlSync("http://localhost:8002") +mc = PryCrawlSync("http://localhost:8005") result = mc.scrape("https://example.com") ``` @@ -383,7 +387,7 @@ POST /v1/commerce/sync } ``` -### Salesforce / HubSpot / Zoho +### Salesforce / HubSpot / ~~Zoho~~ ```json POST /v1/crm/sync diff --git a/cli.py b/cli.py index eda0e24..a88bf20 100755 --- a/cli.py +++ b/cli.py @@ -11,7 +11,6 @@ Usage: pry transform Convert data format pry run [pry.yml] Execute jobs from pry.yml pry serve Start the API server - pry completions Install shell autocomplete pry proxy ... Manage proxy providers and signup """ @@ -231,28 +230,68 @@ def cmd_serve(host="0.0.0.0", port=8005): # nosec B104 os.execvp("uvicorn", ["uvicorn", "api:app", "--host", host, "--port", str(port)]) +def _print_usage() -> None: + """Print the standard Pry usage block (no exit).""" + print(f"{BOLD}Pry v{VERSION}{NC} — Pry open any website. {CYAN}pry.dev{NC}") + print() + print(f" {BOLD}Usage:{NC}") + print(f" {CYAN}pry open {NC} Scrape a URL") + print(f" {CYAN}pry watch {NC} Monitor for changes") + print(f" {CYAN}pry crawl {NC} Crawl a site") + print(f" {CYAN}pry batch {NC} Batch scrape from a file") + print(f" {CYAN}pry parse {NC} Parse a document") + print(f" {CYAN}pry ss {NC} Take a screenshot") + print(f" {CYAN}pry run [pry.yml]{NC} Execute job file") + print(f" {CYAN}pry migrate{NC} Run database migrations") + print(f" {CYAN}pry serve{NC} Start the server") + print(f" {CYAN}pry completions{NC} Install autocomplete") + print(f" {CYAN}pry proxy ...{NC} Manage proxy providers and signup") + print() + print(f" {BOLD}Examples:{NC}") + print(" pry open https://example.com") + print(" pry open https://store.com --json --schema product.json") + print(" pry watch https://site.com --webhook slack://...") + print(" pry crawl https://docs.com --max-pages 20 -o data.json") + print(' pry batch urls.txt --template \'{"price":".price"}\'') + print(" pry run") + print(" pry serve") + print(" pry proxy list") + print(" pry proxy signup brightdata") + print() + print(f" {BOLD}Settings:{NC}") + print(" PRY_URL=http://localhost:8005 (default)") + + def cmd_completions(shell="bash"): - """Install shell autocomplete.""" - script = { - "bash": 'eval "$(_PRY_COMPLETE=bash_source pry)"', - "zsh": 'eval "$(_PRY_COMPLETE=zsh_source pry)"', - "fish": 'eval "$(_PRY_COMPLETE=fish_source pry)"', - }.get(shell, "") - if shell == "bash": - rc = os.path.expanduser("~/.bashrc") - elif shell == "zsh": - rc = os.path.expanduser("~/.zshrc") - elif shell == "fish": - rc = os.path.expanduser("~/.config/fish/config.fish") - else: + """Print shell-autocomplete installation instructions. + + Pry does not currently ship a `_PRY_COMPLETE` env-var handler, so we do + not silently write a broken eval line into the user's shell rc file. + Print copy/paste instructions instead. + """ + if shell not in ("bash", "zsh", "fish"): print(f"{YELLOW}Unknown shell: {shell}. Supported: bash, zsh, fish{NC}") return - dirname = os.path.dirname(rc) - os.makedirs(dirname, exist_ok=True) - with open(rc, "a") as f: - f.write(f"\n# Pry autocomplete\n{script}\n") - print(f"{GREEN}✓{NC} Autocomplete installed for {shell}") - print(f" Restart your shell or run: source {rc}") + print(f"{BOLD}Pry shell autocomplete ({shell}){NC}") + print() + if shell == "bash": + print(" Add the following line to your ~/.bashrc:") + print(f" {CYAN}eval \"$(register-python-argcomplete pry)\"{NC}") + print() + print(" install dependency: pip install argcomplete") + elif shell == "zsh": + print(" Add the following line to your ~/.zshrc:") + print(f" {CYAN}eval \"$(register-python-argcomplete --shell zsh pry)\"{NC}") + print() + print(" install dependency: pip install argcomplete") + else: + print(" Add the following line to ~/.config/fish/config.fish:") + print(f" {CYAN}register-python-argcomplete --shell fish pry | source{NC}") + print() + print(" install dependency: pip install argcomplete") + print() + print(f"{YELLOW}Note:{NC} Pry does not yet wire `_PRY_COMPLETE` itself; once it") + print(" does, this command will switch to a one-shot installer.") def main(): @@ -260,34 +299,7 @@ def main(): cli.main(standalone_mode=False) return if len(sys.argv) < 2: - print(f"{BOLD}Pry v{VERSION}{NC} — Pry open any website. {CYAN}pry.dev{NC}") - print() - print(f" {BOLD}Usage:{NC}") - print(f" {CYAN}pry open {NC} Scrape a URL") - print(f" {CYAN}pry watch {NC} Monitor for changes") - print(f" {CYAN}pry crawl {NC} Crawl a site") - print(f" {CYAN}pry batch {NC} Batch scrape from a file") - print(f" {CYAN}pry parse {NC} Parse a document") - print(f" {CYAN}pry ss {NC} Take a screenshot") - print(f" {CYAN}pry run [pry.yml]{NC} Execute job file") - print(f" {CYAN}pry migrate{NC} Run database migrations") - print(f" {CYAN}pry serve{NC} Start the server") - print(f" {CYAN}pry completions{NC} Install autocomplete") - print(f" {CYAN}pry proxy ...{NC} Manage proxy providers and signup") - print() - print(f" {BOLD}Examples:{NC}") - print(" pry open https://example.com") - print(" pry open https://store.com --json --schema product.json") - print(" pry watch https://site.com --webhook slack://...") - print(" pry crawl https://docs.com --max-pages 20 -o data.json") - print(' pry batch urls.txt --template \'{"price":".price"}\'') - print(" pry run") - print(" pry serve") - print(" pry proxy list") - print(" pry proxy signup brightdata") - print() - print(f" {BOLD}Settings:{NC}") - print(" PRY_URL=http://localhost:8005 (default)") + _print_usage() return cmd = sys.argv[1] @@ -350,7 +362,7 @@ def main(): print(f"Pry v{VERSION}") elif cmd in ("-h", "--help", "help"): - main() + _print_usage() elif cmd == "proxy": if click is None: