docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
499
cli.py
Executable file
499
cli.py
Executable file
|
|
@ -0,0 +1,499 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Pry CLI — pry open any website.
|
||||
|
||||
Usage:
|
||||
pry open <url> Scrape a URL to clean markdown
|
||||
pry watch <url> Monitor a page for changes
|
||||
pry crawl <url> Crawl multiple pages
|
||||
pry batch <file> Batch scrape URLs from a file
|
||||
pry parse <url> Parse a document (PDF, DOCX, image)
|
||||
pry screenshot <url> Take a screenshot
|
||||
pry transform <data> Convert data format
|
||||
pry run [pry.yml] Execute jobs from pry.yml
|
||||
pry serve Start the API server
|
||||
pry completions Install shell autocomplete
|
||||
pry proxy ... Manage proxy providers and signup
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
try:
|
||||
import click
|
||||
except ImportError:
|
||||
click = None # type: ignore[assignment]
|
||||
|
||||
# Color support
|
||||
NC = "\033[0m"
|
||||
RED = "\033[91m"
|
||||
GREEN = "\033[92m"
|
||||
YELLOW = "\033[93m"
|
||||
CYAN = "\033[96m"
|
||||
BOLD = "\033[1m"
|
||||
|
||||
API_DEFAULT = os.getenv("PRY_URL", "http://localhost:8005")
|
||||
VERSION = "3.0.0"
|
||||
|
||||
|
||||
def _api():
|
||||
return os.getenv("PRY_URL", API_DEFAULT)
|
||||
|
||||
|
||||
def _req(method: str, path: str, data: dict = None, timeout=30):
|
||||
import httpx
|
||||
|
||||
url = f"{_api()}{path}"
|
||||
try:
|
||||
if method == "GET":
|
||||
resp = httpx.get(url, timeout=timeout)
|
||||
else:
|
||||
resp = httpx.post(url, json=data or {}, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except httpx.ConnectError:
|
||||
print(f"{RED}✖ Cannot connect to Pry at {_api()}{NC}")
|
||||
print(f" Start it: {CYAN}pry serve{NC}")
|
||||
sys.exit(1)
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 429:
|
||||
print(f"{YELLOW}⚠ Rate limited. Slow down.{NC}")
|
||||
else:
|
||||
print(f"{RED}✖ API error ({e.response.status_code}): {e.response.text[:200]}{NC}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _spinner(label: str):
|
||||
import itertools
|
||||
import threading
|
||||
import time
|
||||
|
||||
stop = False
|
||||
|
||||
def _spin():
|
||||
nonlocal stop
|
||||
for c in itertools.cycle(["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]):
|
||||
if stop:
|
||||
break
|
||||
print(f" {CYAN}{c}{NC} {label}... ", end="\r", flush=True)
|
||||
time.sleep(0.1)
|
||||
|
||||
t = threading.Thread(target=_spin, daemon=True)
|
||||
t.start()
|
||||
|
||||
def _done():
|
||||
nonlocal stop
|
||||
stop = True
|
||||
|
||||
return _done
|
||||
|
||||
|
||||
def cmd_open(url, output_json=False, schema_path=None, timeout=30):
|
||||
"""Scrape a URL and print clean markdown or JSON."""
|
||||
payload = {"url": url, "timeout": timeout}
|
||||
if schema_path:
|
||||
with open(schema_path) as f:
|
||||
payload["jsonSchema"] = json.load(f)
|
||||
s = _spinner(f"Prying open {url[:50]}")
|
||||
data = _req("POST", "/v1/scrape", payload, timeout + 10)
|
||||
print(f"{GREEN}✓{NC} Pry opened {url}\n", end="")
|
||||
if output_json or schema_path:
|
||||
print(json.dumps(data, indent=2))
|
||||
else:
|
||||
md = data.get("data", {}).get("markdown", "")
|
||||
print(md[:100000] if md else f"{YELLOW}(no content extracted){NC}")
|
||||
|
||||
|
||||
def cmd_watch(url, webhook="", interval=3600):
|
||||
"""Register a page for change monitoring."""
|
||||
s = _spinner(f"Registering {url[:50]} for monitoring")
|
||||
data = _req("POST", "/v1/watch", {"url": url, "webhook": webhook, "interval": interval}, 45)
|
||||
if data.get("success"):
|
||||
status = data.get("data", {}).get("status", "registered")
|
||||
print(f"{GREEN}✓{NC} Watching {url} (status: {status})")
|
||||
if webhook:
|
||||
print(f" Webhook: {webhook}")
|
||||
else:
|
||||
print(f"{RED}✖ {data.get('error', 'Watch failed')}{NC}")
|
||||
|
||||
|
||||
def cmd_crawl(url, max_pages=10, output=None, timeout=120):
|
||||
"""Crawl multiple pages from a starting URL."""
|
||||
s = _spinner(f"Crawling {url[:50]} (up to {max_pages} pages)")
|
||||
data = _req("POST", "/v1/crawl", {"url": url, "maxPages": max_pages}, timeout)
|
||||
pages = data.get("data", {}).get("pages", [])
|
||||
print(f"{GREEN}✓{NC} Crawled {len(pages)} page(s) from {url}")
|
||||
if output:
|
||||
with open(output, "w") as f:
|
||||
json.dump(pages, f, indent=2)
|
||||
print(f" Saved to {output}")
|
||||
else:
|
||||
for p in pages[:5]:
|
||||
t = p.get("title", "untitled")[:60]
|
||||
print(f" • {t} ({p.get('method', '?')})")
|
||||
if len(pages) > 5:
|
||||
print(f" ... and {len(pages) - 5} more")
|
||||
|
||||
|
||||
def cmd_batch(filepath, template_str="", timeout=30):
|
||||
"""Scrape URLs from a file using a template."""
|
||||
if not os.path.exists(filepath):
|
||||
print(f"{RED}✖ File not found: {filepath}{NC}")
|
||||
sys.exit(1)
|
||||
template = json.loads(template_str) if template_str else {"body": "body"}
|
||||
s = _spinner(f"Processing {filepath}")
|
||||
data = _req(
|
||||
"POST",
|
||||
"/v1/batch-file",
|
||||
{"filepath": filepath, "template": template, "timeout": timeout},
|
||||
timeout * 5,
|
||||
)
|
||||
results = data.get("data", {}).get("results", [])
|
||||
ok = sum(1 for r in results if r.get("status") == "ok")
|
||||
err = len(results) - ok
|
||||
print(f"{GREEN}✓{NC} Batch complete: {ok} OK, {err} errors from {len(results)} URLs")
|
||||
for r in results[:10]:
|
||||
status = f"{GREEN}✓{NC}" if r.get("status") == "ok" else f"{RED}✖{NC}"
|
||||
print(f" {status} {r.get('url', '')[:70]}")
|
||||
if err:
|
||||
print(f" {YELLOW}Errors: {err}{NC}")
|
||||
|
||||
|
||||
def cmd_parse(url, timeout=60):
|
||||
"""Parse a document to text."""
|
||||
s = _spinner(f"Parsing {url[:50]}")
|
||||
data = _req("POST", "/v1/parse", {"url": url, "timeout": timeout}, timeout + 10)
|
||||
text = data.get("data", {}).get("text", "")
|
||||
fmt = data.get("data", {}).get("format", "unknown")
|
||||
pages = data.get("data", {}).get("pages", 0)
|
||||
print(f"{GREEN}✓{NC} Parsed {fmt} ({pages} pages, {len(text)} chars)")
|
||||
print(text[:50000])
|
||||
|
||||
|
||||
def cmd_screenshot(url, output=None, timeout=30):
|
||||
"""Take a screenshot."""
|
||||
s = _spinner(f"Capturing {url[:50]}")
|
||||
data = _req("POST", "/v1/screenshot", {"url": url}, timeout + 10)
|
||||
b64 = data.get("data", {}).get("screenshot", "")
|
||||
if not b64:
|
||||
print(f"{RED}✖ No screenshot returned{NC}")
|
||||
sys.exit(1)
|
||||
if output:
|
||||
with open(output, "wb") as f:
|
||||
f.write(base64.b64decode(b64))
|
||||
print(f"{GREEN}✓{NC} Screenshot saved to {output}")
|
||||
else:
|
||||
print(f"{GREEN}✓{NC} Screenshot: {len(b64)} bytes (base64)")
|
||||
|
||||
|
||||
def cmd_run(pryfile_path="pry.yml"):
|
||||
"""Execute jobs defined in pry.yml."""
|
||||
if not os.path.exists(pryfile_path):
|
||||
print(f"{RED}✖ No {pryfile_path} found{NC}")
|
||||
print(f" Create one: {CYAN}pry open https://example.com{NC}")
|
||||
sys.exit(1)
|
||||
s = _spinner(f"Running jobs from {pryfile_path}")
|
||||
data = _req("POST", "/v1/run", {"path": pryfile_path}, 120)
|
||||
jobs = data.get("data", {}).get("jobs", [])
|
||||
print(f"{GREEN}✓{NC} Executed {len(jobs)} job(s)")
|
||||
for j in jobs:
|
||||
name = j.get("name", "unnamed")
|
||||
status = j.get("status", "error")
|
||||
if status == "ok":
|
||||
print(
|
||||
f" {GREEN}✓{NC} {name} ({j.get('method', '?')}) — {j.get('content_length', 0)} chars"
|
||||
)
|
||||
else:
|
||||
print(f" {RED}✖{NC} {name} — {j.get('error', 'failed')}{NC}")
|
||||
|
||||
|
||||
def cmd_serve(host="0.0.0.0", port=8005):
|
||||
"""Start the Pry API server."""
|
||||
print(f"{CYAN}🔧 Starting Pry v{VERSION} on {host}:{port}{NC}")
|
||||
print(f" Dashboard: http://localhost:{port}/dashboard")
|
||||
print(f" Health: http://localhost:{port}/health")
|
||||
os.execvp("uvicorn", ["uvicorn", "api:app", "--host", host, "--port", str(port)])
|
||||
|
||||
|
||||
def cmd_completions(shell="bash"):
|
||||
"""Install shell autocomplete."""
|
||||
script = {
|
||||
"bash": 'eval "$(_PRY_COMPLETE=bash_source pry)"',
|
||||
"zsh": 'eval "$(_PRY_COMPLETE=zsh_source pry)"',
|
||||
"fish": 'eval "$(_PRY_COMPLETE=fish_source pry)"',
|
||||
}.get(shell, "")
|
||||
if shell == "bash":
|
||||
rc = os.path.expanduser("~/.bashrc")
|
||||
elif shell == "zsh":
|
||||
rc = os.path.expanduser("~/.zshrc")
|
||||
elif shell == "fish":
|
||||
rc = os.path.expanduser("~/.config/fish/config.fish")
|
||||
else:
|
||||
print(f"{YELLOW}Unknown shell: {shell}. Supported: bash, zsh, fish{NC}")
|
||||
return
|
||||
dirname = os.path.dirname(rc)
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
with open(rc, "a") as f:
|
||||
f.write(f"\n# Pry autocomplete\n{script}\n")
|
||||
print(f"{GREEN}✓{NC} Autocomplete installed for {shell}")
|
||||
print(f" Restart your shell or run: source {rc}")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "proxy" and click is not None:
|
||||
cli.main(standalone_mode=False)
|
||||
return
|
||||
if len(sys.argv) < 2:
|
||||
print(f"{BOLD}Pry v{VERSION}{NC} — Pry open any website. {CYAN}pry.dev{NC}")
|
||||
print()
|
||||
print(f" {BOLD}Usage:{NC}")
|
||||
print(f" {CYAN}pry open <url>{NC} Scrape a URL")
|
||||
print(f" {CYAN}pry watch <url>{NC} Monitor for changes")
|
||||
print(f" {CYAN}pry crawl <url>{NC} Crawl a site")
|
||||
print(f" {CYAN}pry batch <file>{NC} Batch scrape from a file")
|
||||
print(f" {CYAN}pry parse <url>{NC} Parse a document")
|
||||
print(f" {CYAN}pry ss <url>{NC} Take a screenshot")
|
||||
print(f" {CYAN}pry run [pry.yml]{NC} Execute job file")
|
||||
print(f" {CYAN}pry serve{NC} Start the server")
|
||||
print(f" {CYAN}pry completions{NC} Install autocomplete")
|
||||
print(f" {CYAN}pry proxy ...{NC} Manage proxy providers and signup")
|
||||
print()
|
||||
print(f" {BOLD}Examples:{NC}")
|
||||
print(" pry open https://example.com")
|
||||
print(" pry open https://store.com --json --schema product.json")
|
||||
print(" pry watch https://site.com --webhook slack://...")
|
||||
print(" pry crawl https://docs.com --max-pages 20 -o data.json")
|
||||
print(' pry batch urls.txt --template \'{"price":".price"}\'')
|
||||
print(" pry run")
|
||||
print(" pry serve")
|
||||
print(" pry proxy list")
|
||||
print(" pry proxy signup brightdata")
|
||||
print()
|
||||
print(f" {BOLD}Settings:{NC}")
|
||||
print(" PRY_URL=http://localhost:8005 (default)")
|
||||
return
|
||||
|
||||
cmd = sys.argv[1]
|
||||
args = sys.argv[2:]
|
||||
|
||||
try:
|
||||
if cmd == "open" or cmd == "get" or cmd == "scrape":
|
||||
url = args[0] if args else input("URL: ")
|
||||
opts = {"output_json": "--json" in sys.argv, "timeout": 30}
|
||||
if "--schema" in sys.argv:
|
||||
opts["schema_path"] = sys.argv[sys.argv.index("--schema") + 1]
|
||||
if "--timeout" in sys.argv:
|
||||
opts["timeout"] = int(sys.argv[sys.argv.index("--timeout") + 1])
|
||||
cmd_open(url, **opts)
|
||||
|
||||
elif cmd == "watch":
|
||||
url = args[0] if args else input("URL: ")
|
||||
webhook = sys.argv[sys.argv.index("--webhook") + 1] if "--webhook" in sys.argv else ""
|
||||
cmd_watch(url, webhook)
|
||||
|
||||
elif cmd == "crawl":
|
||||
url = args[0] if args else input("URL: ")
|
||||
max_p = (
|
||||
int(sys.argv[sys.argv.index("--max-pages") + 1])
|
||||
if "--max-pages" in sys.argv
|
||||
else 10
|
||||
)
|
||||
out = sys.argv[sys.argv.index("-o") + 1] if "-o" in sys.argv else None
|
||||
cmd_crawl(url, max_p, out)
|
||||
|
||||
elif cmd == "batch":
|
||||
fp = args[0] if args else input("File: ")
|
||||
tmpl = sys.argv[sys.argv.index("--template") + 1] if "--template" in sys.argv else ""
|
||||
cmd_batch(fp, tmpl)
|
||||
|
||||
elif cmd == "parse":
|
||||
url = args[0] if args else input("URL: ")
|
||||
cmd_parse(url)
|
||||
|
||||
elif cmd in ("ss", "screenshot"):
|
||||
url = args[0] if args else input("URL: ")
|
||||
out = sys.argv[sys.argv.index("-o") + 1] if "-o" in sys.argv else None
|
||||
cmd_screenshot(url, out)
|
||||
|
||||
elif cmd == "run":
|
||||
cmd_run(args[0] if args else "pry.yml")
|
||||
|
||||
elif cmd == "serve":
|
||||
port = int(sys.argv[sys.argv.index("--port") + 1]) if "--port" in sys.argv else 8005
|
||||
cmd_serve(port=port)
|
||||
|
||||
elif cmd in ("completions", "autocomplete"):
|
||||
shell = args[0] if args else "bash"
|
||||
cmd_completions(shell)
|
||||
|
||||
elif cmd in ("-v", "--version", "version"):
|
||||
print(f"Pry v{VERSION}")
|
||||
|
||||
elif cmd in ("-h", "--help", "help"):
|
||||
main()
|
||||
|
||||
elif cmd == "proxy":
|
||||
if click is None:
|
||||
print(f"{RED}✖ click is not installed. Run: pip install click{NC}")
|
||||
sys.exit(1)
|
||||
cli.main(standalone_mode=False)
|
||||
return
|
||||
|
||||
else:
|
||||
print(f"{RED}Unknown command: {cmd}{NC}")
|
||||
print(f"Run {CYAN}pry{NC} without arguments for help.")
|
||||
sys.exit(1)
|
||||
|
||||
except IndexError:
|
||||
print(f"{YELLOW}Missing argument for '{cmd}'{NC}")
|
||||
print(f" {CYAN}pry {cmd} --help{NC} for usage.")
|
||||
sys.exit(1)
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n{YELLOW}Interrupted.{NC}")
|
||||
sys.exit(130)
|
||||
|
||||
|
||||
# ── Click-based subcommand: pry proxy ... ──
|
||||
if click is not None:
|
||||
from proxy_manager import ProxyManager
|
||||
|
||||
@click.group()
|
||||
def cli() -> None:
|
||||
"""Pry CLI (click entrypoint for subcommands)."""
|
||||
|
||||
@cli.group()
|
||||
def mcp() -> None:
|
||||
"""Model Context Protocol (MCP) server for AI agent integration."""
|
||||
|
||||
@mcp.command(name="serve")
|
||||
def mcp_serve() -> None:
|
||||
"""Start the MCP server (stdio transport)."""
|
||||
import sys
|
||||
sys.path.insert(0, ".")
|
||||
import asyncio
|
||||
|
||||
from mcp_production import main
|
||||
asyncio.run(main())
|
||||
|
||||
@mcp.command(name="info")
|
||||
def mcp_info() -> None:
|
||||
"""Show MCP server info and configuration."""
|
||||
click.echo("Pry MCP Server v3.0.0")
|
||||
click.echo("")
|
||||
click.echo("Add to Claude Desktop config:")
|
||||
click.echo("""{
|
||||
"mcpServers": {
|
||||
"pry": {
|
||||
"command": "python",
|
||||
"args": ["-m", "mcp_production"]
|
||||
}
|
||||
}
|
||||
}""")
|
||||
click.echo("")
|
||||
click.echo("Or run directly: pry mcp serve")
|
||||
click.echo("Or: python -m mcp_production")
|
||||
|
||||
@cli.group()
|
||||
def proxy() -> None:
|
||||
"""Proxy provider configuration and signup."""
|
||||
|
||||
@proxy.command(name="list")
|
||||
def proxy_list() -> None:
|
||||
"""List all available proxy providers (free + premium)."""
|
||||
pm = ProxyManager()
|
||||
providers = pm.list_providers()
|
||||
click.echo("FREE PROVIDERS:")
|
||||
for p in providers["free"]:
|
||||
click.echo(f" {p['name']:20s} {p['type']:10s} {p['cost']}")
|
||||
click.echo("\nPREMIUM PROVIDERS:")
|
||||
for p in providers["premium"]:
|
||||
click.echo(f" {p['name']:20s} {p['commission']:30s}")
|
||||
click.echo(f" Sign up: {p['signup_url']}")
|
||||
|
||||
@proxy.command(name="signup")
|
||||
@click.argument("provider")
|
||||
def proxy_signup(provider: str) -> None:
|
||||
"""Open signup page for a premium provider (referral link)."""
|
||||
pm = ProxyManager()
|
||||
url = pm.get_signup_link(provider)
|
||||
click.echo(f"Opening: {url}")
|
||||
click.echo(f"(If browser doesn't open, visit: {url})")
|
||||
import webbrowser
|
||||
|
||||
webbrowser.open(url)
|
||||
|
||||
@proxy.command(name="configure")
|
||||
@click.argument("provider")
|
||||
@click.option("--username", default=None)
|
||||
@click.option("--password", default=None)
|
||||
@click.option("--api-key", default=None)
|
||||
@click.option("--proxy-url", default=None)
|
||||
def proxy_configure(
|
||||
provider: str,
|
||||
username: str | None,
|
||||
password: str | None,
|
||||
api_key: str | None,
|
||||
proxy_url: str | None,
|
||||
) -> None:
|
||||
"""Configure credentials for a premium provider."""
|
||||
pm = ProxyManager()
|
||||
creds: dict[str, str] = {}
|
||||
if username:
|
||||
creds["username"] = username
|
||||
if password:
|
||||
creds["password"] = password
|
||||
if api_key:
|
||||
creds["api_key"] = api_key
|
||||
if proxy_url:
|
||||
creds["proxy_url"] = proxy_url
|
||||
if not creds:
|
||||
click.echo(
|
||||
"Need at least one credential "
|
||||
"(--username, --password, --api-key, or --proxy-url)"
|
||||
)
|
||||
return
|
||||
result = pm.select_provider(provider, creds)
|
||||
if result["success"]:
|
||||
click.echo(f"Configured {provider}")
|
||||
else:
|
||||
click.echo(f"Error: {result.get('error', 'unknown')}")
|
||||
|
||||
@proxy.command(name="test")
|
||||
@click.option("--url", default="https://httpbin.org/ip", help="URL to test against")
|
||||
def proxy_test(url: str) -> None:
|
||||
"""Test the active proxy connection."""
|
||||
pm = ProxyManager()
|
||||
proxy_url = pm.get_proxy_url()
|
||||
if not proxy_url:
|
||||
click.echo("No active proxy configured. Using direct connection.")
|
||||
result = pm.test_proxy(proxy_url, url) if proxy_url else {"working": True, "latency": 0}
|
||||
click.echo(f"Working: {result['working']}")
|
||||
click.echo(f"Latency: {result.get('latency', 'N/A')}s")
|
||||
if result.get("ip"):
|
||||
click.echo(f"IP: {result['ip'][:60]}")
|
||||
|
||||
@proxy.command(name="status")
|
||||
def proxy_status() -> None:
|
||||
"""Show current proxy configuration."""
|
||||
pm = ProxyManager()
|
||||
c = pm.active_config
|
||||
click.echo(f"Active provider: {c.provider}")
|
||||
click.echo(f"Type: {c.proxy_type}")
|
||||
click.echo(f"Geo: {c.geo}")
|
||||
click.echo(f"Auto-rotate: {c.auto_rotate}")
|
||||
creds = list(pm.credentials.keys())
|
||||
if creds:
|
||||
click.echo(f"Configured: {', '.join(creds)}")
|
||||
|
||||
@proxy.command(name="recommend")
|
||||
@click.option("--error", "last_error", default="", help="Last error message")
|
||||
def proxy_recommend(last_error: str) -> None:
|
||||
"""Get proxy recommendation after a scrape block."""
|
||||
pm = ProxyManager()
|
||||
rec = pm.get_recommendation(last_error)
|
||||
click.echo(json.dumps(rec, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue