style: ruff format
Some checks failed
CI / lint (pull_request) Successful in 33s
CI / typecheck (pull_request) Failing after 1m42s
CI / test (pull_request) Failing after 2m33s
CI / security (pull_request) Failing after 39s
CI / gitleaks (pull_request) Successful in 36s
CI / commitlint (pull_request) Failing after 10s

This commit is contained in:
Crypto Rug Munch 2026-07-03 18:12:36 +02:00
parent 68722b2c22
commit 2f1eec2f78
58 changed files with 166 additions and 395 deletions

333
api.py
View file

@ -388,51 +388,21 @@ class PryHttpMiddleware:
app.add_middleware(PryHttpMiddleware)
# ── Stats ──
# ── Costing ──
# ── Freshness ──
# ── Parse (Documents) ──
# ── Automate (Browser) ──
# ── Vision — analyze an image via free OpenRouter vision models ───────────────
# Free models default to google/gemma-4-31b-it:free. Accepts:
# - image (base64 data URI or raw base64)
@ -445,113 +415,39 @@ app.add_middleware(PryHttpMiddleware)
# Auto-fallback: if primary model 429s, tries other free models in order.
# ── Sessions ──────────────────────────────────────────────────────────────────
# ── Email Inbox Scraping ──
# ── Integrations ──
# ── Alerts ──
# ── Jobs ──
# ── Config ──
# ── Win 3: WebSocket streaming ──
# ── Win 5: Batch from file + template ──
# ── Win 6: Browser recorder ──
# ── Win 8: Multi-format transform ──
# ── Win 1: Pryfile execution ──
# ── Win 10: Health dashboard HTML ──
@ -561,120 +457,42 @@ app.add_middleware(PryHttpMiddleware)
# ── Win 2: Circuit Breaker ──
# ── Win 3: Stable Extraction ──
# ── Pipeline hooks ──
# ── Win 4: Data Pipeline ──
# ── Win 5: Shareable Results ──
# ── Compliance ──
# ── GDPR Compliance Portal ──
# ── AI Training Data Pipeline ──
# ── Monitoring (cron-based content change detection with AI judging) ──
# ── Structure Monitor ──
# ── Intelligence (Competitive Intelligence Engine) ──
# ── Quality ──
# ── Reconciliation ──
# ── Auth ──
# (split into routers/auth.py on the api-router-split refactor)
@ -733,201 +551,50 @@ app.include_router(webhooks_router)
app.include_router(x402_router)
# ── Review ──
# ── Commerce Sync ──
# ── CRM Sync ──
# ── Pipeline Builder Endpoints ──
# ── Agency ──
# ── SEO Content Change Monitor ──
# ── Reports ──
# ── Enrichment ──
# ── AI Agent Integration ──
# ── Proxy Provider & Affiliate Signup Flow ──
# ── Advanced Scraping (TLS, GraphQL, Schema.org, WebSocket) ──
# ── Advanced: Cookie Warming, PDF, OCR, Dedup, Behavior ──
# ── x402 payment processing ──
# ── Actor marketplace ──
# ── Webhook delivery ──
if __name__ == "__main__":
uvicorn.run(app, host=settings.host, port=settings.port)

View file

@ -31,6 +31,7 @@ from pry_mcp.notifications import (
logger = logging.getLogger(__name__)
def make_fallback_server(
base_url: str = DEFAULT_BASE_URL,
tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]]

View file

@ -59,6 +59,7 @@ try:
except ImportError:
_has_mcp_sdk = False
def create_server() -> Any:
"""Create the MCP server (uses official SDK if available, fallback otherwise)."""
if _has_mcp_sdk:

View file

@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Advanced"])
@router.post(
"/v1/tls/impersonate",
tags=["Advanced"],

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Agency"])
@router.post("/v1/agency/create", tags=["Agency"], summary="Create a white-label agency profile")
async def create_agency_endpoint(
name: str = Body(...),
@ -67,7 +68,9 @@ async def update_branding(
return {"success": result["success"], "data": result}
@router.post("/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account")
@router.post(
"/v1/agency/{agency_id}/clients", tags=["Agency"], summary="Create a client sub-account"
)
async def create_client_endpoint(
agency_id: str,
client_name: str = Body(...),
@ -93,7 +96,9 @@ async def list_clients_endpoint(agency_id: str) -> dict[str, Any]:
return {"success": True, "data": {"clients": clients, "total": len(clients)}}
@router.get("/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics")
@router.get(
"/v1/agency/{agency_id}/analytics", tags=["Agency"], summary="Get agency usage analytics"
)
async def get_analytics(agency_id: str) -> dict[str, Any]:
"""Get aggregate usage analytics for an agency."""
from agency import get_agency_analytics

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["AI"])
@router.get("/v1/ai/gpt-manifest", tags=["AI"], summary="Get GPT Action manifest for ChatGPT")
async def get_gpt_manifest() -> JSONResponse:
"""Get the GPT Action manifest for ChatGPT integration.

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Alerts"])
@router.post("/v1/alert/send", tags=["Alerts"], summary="Send an alert to any channel")
async def send_alert_endpoint(
channel: str = Body(...),

View file

@ -27,6 +27,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Analysis"])
@router.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs")
async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]:
"""Scrape two URLs and compare their content. Shows additions, deletions, changes."""

View file

@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Automation"])
@router.post("/v1/automate", tags=["Automation"], summary="Execute browser automation steps")
async def automate(request: AutomateRequest) -> dict[str, Any]:
"""Execute browser automation steps. Sessions persist for login flows."""

View file

@ -20,7 +20,10 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Batch"])
@router.post("/v1/batch-file", tags=["Batch"], summary="Batch scrape URLs from a file with templates")
@router.post(
"/v1/batch-file", tags=["Batch"], summary="Batch scrape URLs from a file with templates"
)
async def batch_from_file(request: BatchFileRequest) -> dict[str, Any]:
bp = BatchProcessor()
results = await bp.from_file(

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Circuit Breaker"])
@router.post("/v1/breaker/status", tags=["Circuit Breaker"], summary="Get circuit breaker status")
async def breaker_status() -> dict[str, Any]:
return {

View file

@ -19,7 +19,10 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Commerce"])
@router.post("/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify")
@router.post(
"/v1/commerce/sync", tags=["Commerce"], summary="Sync products to WooCommerce or Shopify"
)
async def sync_commerce(
platform: str = Body(...),
products: list[dict[str, Any]] = Body(...),
@ -58,7 +61,9 @@ async def sync_commerce(
return {"success": result["success"], "data": result}
@router.get("/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms")
@router.get(
"/v1/commerce/platforms", tags=["Commerce"], summary="List supported commerce platforms"
)
async def list_commerce_platforms() -> dict[str, Any]:
"""List supported e-commerce platforms and their credential requirements."""
return {

View file

@ -17,7 +17,10 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Compliance"])
@router.post("/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL")
@router.post(
"/v1/compliance/check", tags=["Compliance"], summary="Run full compliance check on a URL"
)
async def compliance_check(url: str = Body(...)) -> dict[str, Any]:
"""Run a full legal compliance check on a target URL.
@ -35,7 +38,9 @@ async def compliance_check(url: str = Body(...)) -> dict[str, Any]:
return {"success": True, "data": result}
@router.get("/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation")
@router.get(
"/v1/compliance/check", tags=["Compliance"], summary="Get compliance check documentation"
)
async def compliance_docs() -> dict[str, Any]:
"""Get information about the compliance check endpoint."""
return {

View file

@ -34,7 +34,9 @@ async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]:
return {"success": True, "data": result}
@router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests")
@router.post(
"/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests"
)
async def enable_tor() -> dict[str, Any]:
"""Enable Tor routing for all requests."""
result = settings.enable_tor()

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Costing"])
@router.get("/v1/costing/dashboard", tags=["Costing"], summary="Get cost analytics dashboard")
async def cost_dashboard() -> dict[str, Any]:
"""Get the full cost analytics dashboard.

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["CRM"])
@router.post("/v1/crm/sync", tags=["CRM"], summary="Sync scraped data to CRM")
async def sync_crm(
platform: str = Body(...),

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Dashboard"])
@router.get("/dashboard", tags=["Dashboard"], summary="Pry health and performance dashboard")
async def dashboard() -> HTMLResponse:
cache_stats = cache.stats()

View file

@ -17,7 +17,10 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Email"])
@router.post("/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)")
@router.post(
"/v1/email/scrape", tags=["Email"], summary="Extract data from an email (subject + body)"
)
async def scrape_email(
subject: str = Body(""),
body: str = Body(""),
@ -56,7 +59,9 @@ async def fetch_gmail(
return {"success": result.get("success", False), "data": result}
@router.post("/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox")
@router.post(
"/v1/email/outlook", tags=["Email"], summary="Fetch and extract data from Outlook inbox"
)
async def fetch_outlook(
access_token: str = Body(...),
max_results: int = Body(20),

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Enrichment"])
@router.post(
"/v1/enrich",
tags=["Enrichment"],

View file

@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Execute"])
@router.post("/v1/run", tags=["Execute"], summary="Execute a Pryfile")
async def run_pryfile(path: str = Body("pry.yml")) -> dict[str, Any]:
from pryfile import Pryfile

View file

@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Export"])
@router.post("/v1/export", tags=["Export"], summary="Export scraped content in multiple formats")
async def export_data(url: str = Body(...), format: str = "json") -> dict[str, Any]:
"""Export scraped content in multiple formats: json, csv, txt, rss.

View file

@ -26,6 +26,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Extraction"])
@router.post(
"/v1/extract-table", tags=["Extraction"], summary="Extract HTML tables as structured data"
)
@ -122,7 +123,9 @@ async def analyze_seo(url: str = Body(...)) -> dict[str, Any]:
}
@router.post("/v1/schema", tags=["Extraction"], summary="Extract Schema.org/JSON-LD structured data")
@router.post(
"/v1/schema", tags=["Extraction"], summary="Extract Schema.org/JSON-LD structured data"
)
async def extract_schema(url: str = Body(...)) -> dict[str, Any]:
"""Extract Schema.org/JSON-LD structured data from a page."""
client = await get_client()
@ -211,7 +214,9 @@ async def extract_css(
}
@router.post("/v1/extract/llm", tags=["Extraction"], summary="Extract with LLM + chunking strategies")
@router.post(
"/v1/extract/llm", tags=["Extraction"], summary="Extract with LLM + chunking strategies"
)
async def extract_llm(
url: str = Body(...),
instruction: str = Body("Extract all key information from this content."),

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Freshness"])
@router.post(
"/v1/freshness/check",
tags=["Freshness"],
@ -63,7 +64,9 @@ async def freshness_frequency(
return {"success": True, "data": result}
@router.get("/v1/freshness/dashboard", tags=["Freshness"], summary="Get content staleness dashboard")
@router.get(
"/v1/freshness/dashboard", tags=["Freshness"], summary="Get content staleness dashboard"
)
async def freshness_dashboard() -> dict[str, Any]:
"""Get the content staleness dashboard.

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["GDPR"])
@router.post("/v1/gdpr/consent", tags=["GDPR"], summary="Record user consent for data processing")
async def record_consent_endpoint(
user_id: str = Body(...),

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Integrations"])
@router.get(
"/v1/destinations",
tags=["Integrations"],

View file

@ -17,7 +17,10 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Intelligence"])
@router.post("/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot")
@router.post(
"/v1/intel/snapshot", tags=["Intelligence"], summary="Record a competitor data snapshot"
)
async def record_intel_snapshot(
competitor_id: str = Body(...),
competitor_name: str = Body(...),

View file

@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Jobs"])
@router.get("/v1/job/{job_id}", tags=["Jobs"], summary="Get async job status and result")
async def get_job(job_id: str) -> dict[str, Any]:
"""Get async job status and result."""

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Marketplace"])
@router.post(
"/v1/actors/create",
tags=["Marketplace"],

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Monitoring"])
@router.post("/v1/monitor", tags=["Monitoring"], summary="Create a scheduled content monitor")
async def create_monitor_endpoint(
name: str = Body(...),

View file

@ -23,7 +23,10 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Parsing"])
@router.post("/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)")
@router.post(
"/v1/parse", tags=["Parsing"], summary="Parse a document (PDF, DOCX, image, CSV, JSON)"
)
async def parse_document(request: ParseRequest) -> dict[str, Any]:
"""Parse a document (PDF, DOCX, image, CSV, JSON) to text."""
try:

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Pipeline"])
@router.post("/v1/pipeline/hook", tags=["Pipeline"], summary="Register a hook function")
async def register_hook(
hook_point: str = Body(...),

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Pipelines"])
@router.get(
"/v1/pipelines/steps", tags=["Pipelines"], summary="List all available pipeline step types"
)

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Proxy"])
@router.get("/v1/proxy/providers", tags=["Proxy"], summary="List all available proxy providers")
async def list_proxy_providers() -> dict[str, Any]:
"""List free + premium proxy providers with affiliate details."""
@ -25,7 +26,9 @@ async def list_proxy_providers() -> dict[str, Any]:
return {"success": True, "data": ProxyManager().list_providers()}
@router.post("/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider")
@router.post(
"/v1/proxy/signup", tags=["Proxy"], summary="Open affiliate signup link for a provider"
)
async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]:
"""Get the affiliate signup URL for a proxy provider.
@ -39,7 +42,9 @@ async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]:
return {"success": True, "data": {"signup_url": url, "provider": provider}}
@router.get("/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals")
@router.get(
"/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals"
)
async def list_proxy_referrals() -> dict[str, Any]:
"""Return the curated proxy provider affiliate catalog (proxy_referrals.py).
@ -126,7 +131,9 @@ async def proxy_status() -> dict[str, Any]:
}
@router.post("/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block")
@router.post(
"/v1/proxy/recommend", tags=["Proxy"], summary="Get proxy recommendation after a block"
)
async def proxy_recommend(last_error: str = Body("")) -> dict[str, Any]:
"""After a scrape fails with anti-bot detection, get a recommendation
for which premium proxy provider to sign up with."""

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Quality"])
@router.post(
"/v1/quality/check", tags=["Quality"], summary="Run data quality check on extraction results"
)

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Reconciliation"])
@router.post(
"/v1/reconcile",
tags=["Reconciliation"],

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Recorder"])
@router.post("/v1/record/start", tags=["Recorder"], summary="Start recording browser actions")
async def start_recording(session_id: str = Body(...)) -> dict[str, Any]:
recorder.start(session_id)

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Referrals"])
@router.get(
"/v1/referrals/catalog",
tags=["Referrals"],

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Reports"])
@router.post(
"/v1/reports/generate",
tags=["Reports"],

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Review"])
@router.post("/v1/review/submit", tags=["Review"], summary="Submit extracted data for human review")
async def review_submit(
data: dict[str, Any] = Body(...),

View file

@ -264,7 +264,9 @@ async def crawl(request: CrawlRequest) -> dict[str, Any]:
{
"max_pages": request.maxPages,
"max_depth": request.maxDepth,
"timeout": request.scrapeOptions.get("timeout", 60) if request.scrapeOptions else 60,
"timeout": request.scrapeOptions.get("timeout", 60)
if request.scrapeOptions
else 60,
},
)
return {"success": True, "data": {"id": "sync", "url": request.url, "pages": pages}}

View file

@ -24,6 +24,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Scraping"])
@router.post(
"/v1/ultimate-scrape", tags=["Scraping"], summary="Scrape with 10-tier anti-bot fallback system"
)

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["SEO"])
@router.post("/v1/seo/analyze", tags=["SEO"], summary="Analyze SEO elements from a URL")
async def seo_analyze(url: str = Body(...)) -> dict[str, Any]:
"""Analyze all SEO elements from a URL.

View file

@ -21,6 +21,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Sessions"])
@router.post("/v1/session/create", tags=["Sessions"], summary="Create a persistent browser session")
async def create_session(url: str = Body(...), persist: bool = True) -> dict[str, Any]:
"""Create a persistent browser session."""

View file

@ -23,6 +23,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Share"])
@router.post("/v1/share", tags=["Share"], summary="Share scraped content via link")
async def share_scrape(data: dict[str, Any] = Body(...)) -> dict[str, Any]:
url = data.get("url", "")

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Stats"])
@router.get("/v0/stats", tags=["Stats"], summary="Get cache, rate limiter, and session stats")
async def stats() -> dict[str, Any]:
return {

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Structure"])
@router.post(
"/v1/structure/check",
tags=["Structure"],

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["System"])
@router.get("/metrics", tags=["System"], summary="Prometheus metrics", include_in_schema=False)
async def metrics() -> Response:
"""Expose Prometheus metrics for scraping."""

View file

@ -19,6 +19,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Training"])
@router.post(
"/v1/training/classify-license",
tags=["Training"],
@ -36,7 +37,9 @@ async def classify_license_endpoint(text: str = Body(...)) -> dict[str, Any]:
return {"success": True, "data": result}
@router.post("/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content")
@router.post(
"/v1/training/clean", tags=["Training"], summary="Strip PII and copyright from content"
)
async def clean_content(
text: str = Body(...),
strip_names: bool = Body(False),

View file

@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Transform"])
@router.post("/v1/transform", tags=["Transform"], summary="Transform data to multiple formats")
async def transform_data(
data: list[dict[str, Any]] = Body(...), format: str = "csv"

View file

@ -18,6 +18,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Untagged"])
@router.websocket("/v1/stream/{job_id}")
async def stream_endpoint(websocket: WebSocket, job_id: str = "live") -> None:
await websocket.accept()

View file

@ -26,6 +26,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Vision"])
@router.post("/v1/vision", tags=["Vision"], summary="Analyze an image with a free vision model")
async def vision(
question: str = Body("Describe what is visible in this image.", embed=True),

View file

@ -17,6 +17,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["Webhooks"])
@router.post(
"/v1/webhooks/test",
tags=["Webhooks"],

View file

@ -20,6 +20,7 @@ logger = logging.getLogger(__name__)
router = APIRouter(tags=["x402"])
@router.get("/v1/x402/pricing", tags=["x402"], summary="Get x402 pricing for all paid operations")
async def x402_pricing() -> dict[str, Any]:
"""Get the price list for pay-per-scrape operations."""

View file

@ -236,7 +236,9 @@ class PrySettings(BaseSettings):
proxies.append(f"socks5://{self.tor_socks5_host}:{self.tor_socks5_port}")
if self.proxy_enabled and self.proxy_url:
if self.proxy_username:
netloc = self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url
netloc = (
self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url
)
proxies.append(
f"{self.proxy_type}://{self.proxy_username}:{self.proxy_password}@{netloc}"
)

View file

@ -17,8 +17,8 @@ from retry import Jitter, RetryConfig, async_retry
def _valid_html(text: str = "content") -> str:
body = f"<p>{text}</p>" * 40
return (
"<!DOCTYPE html><html lang=\"en\"><head><title>T</title>"
"<meta charset=\"utf-8\"></head>"
'<!DOCTYPE html><html lang="en"><head><title>T</title>'
'<meta charset="utf-8"></head>'
f"<body>{body}</body></html>"
)
@ -152,7 +152,9 @@ class TestCircuitBreakerIntegration:
monkeypatch.setattr(resilience.time, "time", lambda: future)
fn = AsyncMock(return_value="ok")
cfg = RetryConfig(max_retries=1, circuit_breaker="ollama_int", base_delay=0.01, jitter=Jitter.NONE)
cfg = RetryConfig(
max_retries=1, circuit_breaker="ollama_int", base_delay=0.01, jitter=Jitter.NONE
)
result = await async_retry(fn, config=cfg)
assert result == "ok"
@ -202,7 +204,9 @@ class TestDomainTierMemoryWithRetry:
async def test_domain_memory_used_on_retry(self, fresh_registry, scraper_with_mocks):
"""After domain learns best tier, prefers it."""
s, _mocks = scraper_with_mocks
fresh_registry.record_domain_tier_result("example.com", "archive", success=True, latency=0.1)
fresh_registry.record_domain_tier_result(
"example.com", "archive", success=True, latency=0.1
)
html = _valid_html("archive content")
_mocks["_tier_archive"].return_value = html
_mocks["_tier_direct"].return_value = _valid_html("direct")
@ -242,7 +246,9 @@ class TestGracefulDegradation:
raise RuntimeError(result.get("error", "scrape failed"))
return result
cfg = RetryConfig(max_retries=1, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE)
cfg = RetryConfig(
max_retries=1, circuit_breaker=name, base_delay=0.01, jitter=Jitter.NONE
)
with pytest.raises(RuntimeError):
await async_retry(scrape_and_raise, "https://example.com/page", config=cfg)
@ -255,11 +261,13 @@ class TestRetrySettings:
def test_retry_config_from_settings(self) -> None:
from settings import PrySettings
settings = PrySettings()
assert settings.retry_backoff == "exponential"
def test_api_dict_includes_backoff(self) -> None:
from settings import PrySettings
settings = PrySettings()
api_dict = settings.to_api_dict()
assert "retry" in api_dict

View file

@ -16,8 +16,8 @@ from resilience import CircuitBreaker, CircuitState, DomainTierMemory, Resilienc
def _valid_html(text: str) -> str:
body = f"<p>{text}</p>" * 40
return (
"<!DOCTYPE html><html lang=\"en\"><head><title>T</title>"
"<meta charset=\"utf-8\"></head>"
'<!DOCTYPE html><html lang="en"><head><title>T</title>'
'<meta charset="utf-8"></head>'
f"<body>{body}</body></html>"
)
@ -159,10 +159,7 @@ class TestUltimateScraperResilience:
fresh_registry.record_service_result("flaresolverr", False)
html = _valid_html("archive content")
mocks = {
name: AsyncMock(return_value=None)
for name in _TIER_NAMES
}
mocks = {name: AsyncMock(return_value=None) for name in _TIER_NAMES}
mocks["_tier_archive"].return_value = html
with ExitStack() as stack:
for name in _TIER_NAMES:
@ -201,9 +198,7 @@ class TestUltimateScraperResilience:
for name in _TIER_NAMES:
stack.enter_context(patch.object(scraper, name, new=mocks[name]))
await scraper.scrape("https://example.com/page")
assert fresh_registry.best_tier_for_domain(
"example.com", ["direct", "archive"]
) == "direct"
assert fresh_registry.best_tier_for_domain("example.com", ["direct", "archive"]) == "direct"
@pytest.mark.anyio
async def test_records_tier_failure_no_success_memory(self, fresh_registry):
@ -214,6 +209,4 @@ class TestUltimateScraperResilience:
stack.enter_context(patch.object(scraper, name, new=mocks[name]))
result = await scraper.scrape("https://example.com/page")
assert result["status"] == "error"
assert fresh_registry.best_tier_for_domain(
"example.com", ["direct", "archive"]
) is None
assert fresh_registry.best_tier_for_domain("example.com", ["direct", "archive"]) is None

View file

@ -30,12 +30,21 @@ def _clear_registry() -> None:
_TIER_ALL = [
"_tier_direct", "_tier_tls_fingerprint", "_tier_cloudscraper",
"_tier_flaresolverr", "_tier_undetected", "_tier_camoufox",
"_tier_playwright", "_tier_googlebot", "_tier_premium_proxy",
"_tier_archive", "_tier_google_cache", "_tier_tor",
"_tier_direct",
"_tier_tls_fingerprint",
"_tier_cloudscraper",
"_tier_flaresolverr",
"_tier_undetected",
"_tier_camoufox",
"_tier_playwright",
"_tier_googlebot",
"_tier_premium_proxy",
"_tier_archive",
"_tier_google_cache",
"_tier_tor",
]
def test_ultimate_init() -> None:
s = UltimateScraper()
assert s is not None
@ -132,17 +141,19 @@ async def test_tier_camoufox_failure() -> None:
async def test_scrape_falls_through_all_tiers() -> None:
_clear_registry()
s = UltimateScraper()
with patch.object(s, "_tier_direct", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_tls_fingerprint", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_cloudscraper", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_flaresolverr", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_undetected", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_camoufox", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_playwright", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_googlebot", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_archive", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_google_cache", new=AsyncMock(return_value=None)), \
patch.object(s, "_tier_tor", new=AsyncMock(return_value=None)):
with (
patch.object(s, "_tier_direct", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_tls_fingerprint", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_cloudscraper", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_flaresolverr", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_undetected", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_camoufox", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_playwright", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_googlebot", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_archive", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_google_cache", new=AsyncMock(return_value=None)),
patch.object(s, "_tier_tor", new=AsyncMock(return_value=None)),
):
result = await s.scrape("http://example.com")
assert result["status"] == "error"

View file

@ -25,15 +25,15 @@ logger = logging.getLogger(__name__)
# Private/reserved IP ranges (RFC 1918, RFC 6598, RFC 4193, link-local, loopback)
_PRIVATE_NETS: list[str] = [
"127.0.0.0/8", # loopback
"10.0.0.0/8", # RFC 1918
"172.16.0.0/12", # RFC 1918
"192.168.0.0/16", # RFC 1918
"100.64.0.0/10", # RFC 6598 (CGNAT)
"169.254.0.0/16", # link-local
"fc00::/7", # RFC 4193 (ULA)
"fe80::/10", # link-local IPv6
"::1/128", # IPv6 loopback
"127.0.0.0/8", # loopback
"10.0.0.0/8", # RFC 1918
"172.16.0.0/12", # RFC 1918
"192.168.0.0/16", # RFC 1918
"100.64.0.0/10", # RFC 6598 (CGNAT)
"169.254.0.0/16", # link-local
"fc00::/7", # RFC 4193 (ULA)
"fe80::/10", # link-local IPv6
"::1/128", # IPv6 loopback
]
# Common internal hostnames that suggest SSRF targeting