diff --git a/account_manager.py b/account_manager.py index c0295af..962c6ff 100644 --- a/account_manager.py +++ b/account_manager.py @@ -1,4 +1,4 @@ -import httpx + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -11,10 +11,12 @@ import logging import os import time from datetime import UTC, datetime -from pathlib import Path from typing import Any +import httpx + from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) ACCOUNTS_DIR = PRY_DATA_DIR / "accounts" @@ -24,14 +26,27 @@ ACCOUNTS_DIR.mkdir(parents=True, exist_ok=True) class AccountPool: """Manage pool of registered accounts with session persistence.""" - def store(self, site: str, credentials: dict[str, Any], profile_id: str = "", metadata: dict | None = None) -> str: + def store( + self, + site: str, + credentials: dict[str, Any], + profile_id: str = "", + metadata: dict | None = None, + ) -> str: import uuid + account_id = uuid.uuid4().hex[:12] account = { - "id": account_id, "site": site, "credentials": credentials, - "profile_id": profile_id, "metadata": metadata or {}, - "status": "active", "created_at": datetime.now(UTC).isoformat(), - "last_used": None, "use_count": 0, "errors": [], + "id": account_id, + "site": site, + "credentials": credentials, + "profile_id": profile_id, + "metadata": metadata or {}, + "status": "active", + "created_at": datetime.now(UTC).isoformat(), + "last_used": None, + "use_count": 0, + "errors": [], } path = ACCOUNTS_DIR / f"{site}_{account_id}.json" path.write_text(json.dumps(account, indent=2)) @@ -51,6 +66,7 @@ class AccountPool: if not accounts: return None import random + return random.choice(accounts) def mark_error(self, account_id: str, error: str) -> None: @@ -90,15 +106,23 @@ class AccountPool: class ProxyScorer: """Score and rank proxies for reliability.""" - async def test_proxy(self, proxy_url: str, test_url: str = "https://httpbin.org/ip", timeout: int = 10) -> dict[str, Any]: + async def test_proxy( + self, proxy_url: str, test_url: str = "https://httpbin.org/ip", timeout: int = 10 + ) -> dict[str, Any]: from client import get_client + client = await get_client() start = time.time() try: resp = await client.get(test_url, timeout=timeout) elapsed = time.time() - start - return {"proxy": proxy_url, "working": resp.is_success, "latency": round(elapsed, 2), - "status": resp.status_code, "ip": resp.text[:50] if resp.is_success else ""} + return { + "proxy": proxy_url, + "working": resp.is_success, + "latency": round(elapsed, 2), + "status": resp.status_code, + "ip": resp.text[:50] if resp.is_success else "", + } except (httpx.HTTPError, httpx.RequestError) as e: return {"proxy": proxy_url, "working": False, "error": str(e)[:80]} diff --git a/actor_marketplace.py b/actor_marketplace.py index 8b3e4dc..03d1cda 100644 --- a/actor_marketplace.py +++ b/actor_marketplace.py @@ -1,23 +1,21 @@ """Pry — Actor Marketplace (Apify-style). Pre-built scrapers that can be subscribed to and run on schedule. Users can publish their own actors. We earn from usage.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging -import os import uuid from datetime import UTC, datetime from enum import StrEnum -from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) ACTOR_DIR = PRY_DATA_DIR / "actors" @@ -33,10 +31,18 @@ class ActorVisibility(StrEnum): class Actor: """A reusable scraper actor that can be run on demand or schedule.""" - def __init__(self, actor_id: str, name: str, description: str, - template_id: str = "", code: str = "", - price_per_run: float = 0.0, visibility: ActorVisibility = ActorVisibility.PRIVATE, - schedule_cron: str = "", tags: list[str] | None = None): + def __init__( + self, + actor_id: str, + name: str, + description: str, + template_id: str = "", + code: str = "", + price_per_run: float = 0.0, + visibility: ActorVisibility = ActorVisibility.PRIVATE, + schedule_cron: str = "", + tags: list[str] | None = None, + ): self.actor_id = actor_id self.name = name self.description = description @@ -93,13 +99,29 @@ class ActorMarketplace: except OSError as e: logger.warning("actor_save_failed", extra={"actor_id": actor.actor_id, "error": str(e)}) - def create(self, name: str, description: str, template_id: str = "", - code: str = "", price_per_run: float = 0.0, - visibility: ActorVisibility = ActorVisibility.PRIVATE, - schedule_cron: str = "", tags: list[str] | None = None, - author: str = "") -> Actor: - actor = Actor(uuid.uuid4().hex[:12], name, description, template_id, code, - price_per_run, visibility, schedule_cron, tags) + def create( + self, + name: str, + description: str, + template_id: str = "", + code: str = "", + price_per_run: float = 0.0, + visibility: ActorVisibility = ActorVisibility.PRIVATE, + schedule_cron: str = "", + tags: list[str] | None = None, + author: str = "", + ) -> Actor: + actor = Actor( + uuid.uuid4().hex[:12], + name, + description, + template_id, + code, + price_per_run, + visibility, + schedule_cron, + tags, + ) actor.author = author self.actors[actor.actor_id] = actor self._save(actor) @@ -125,9 +147,14 @@ class ActorMarketplace: self._save(actor) if actor.template_id: from template_engine import execute_template + url = (inputs or {}).get("url", "") if not url: return {"success": False, "error": "Missing 'url' input"} result = await execute_template(actor.template_id, url) return {"success": True, "actor_id": actor_id, "data": result.get("data", {})} - return {"success": True, "actor_id": actor_id, "message": "Custom actor code not yet executed"} + return { + "success": True, + "actor_id": actor_id, + "message": "Custom actor code not yet executed", + } diff --git a/advanced.py b/advanced.py index 2ea2289..db0aca6 100644 --- a/advanced.py +++ b/advanced.py @@ -251,5 +251,3 @@ class PryAdvanced: level = "very difficult" return {"score": round(score, 1), "level": level, "words": words, "sentences": sentences} - - diff --git a/agency.py b/agency.py index ff274ff..962ac09 100644 --- a/agency.py +++ b/agency.py @@ -1,21 +1,20 @@ """Pry — White-Label Agency Dashboard. Multi-tenant reseller platform: custom branding, client management, sub-accounts.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging import os import uuid from datetime import UTC, datetime -from pathlib import Path from typing import Any, cast +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) AGENCY_DIR = PRY_DATA_DIR / "agency" diff --git a/alerter.py b/alerter.py index 4af8b7a..f73b4a0 100644 --- a/alerter.py +++ b/alerter.py @@ -1,4 +1,4 @@ -import httpx + """Pry — Multi-Channel Alerting System. SMS, Email, Microsoft Teams, Discord, Telegram alerts.""" @@ -11,6 +11,8 @@ SMS, Email, Microsoft Teams, Discord, Telegram alerts.""" import logging from typing import Any +import httpx + logger = logging.getLogger(__name__) diff --git a/anomaly.py b/anomaly.py index 97d4964..6f464ea 100644 --- a/anomaly.py +++ b/anomaly.py @@ -82,7 +82,9 @@ class AnomalyDetector: if isinstance(v, (int, float)) and not isinstance(v, bool): common.add(k) for r in records[1:]: - rkeys = {k for k, v in r.items() if isinstance(v, (int, float)) and not isinstance(v, bool)} + rkeys = { + k for k, v in r.items() if isinstance(v, (int, float)) and not isinstance(v, bool) + } common &= rkeys return list(common) @@ -139,16 +141,14 @@ class AnomalyDetector: if current_dow in dow_values and len(dow_values[current_dow]) >= 2: dow_mean = statistics.mean(dow_values[current_dow]) dow_stdev = ( - statistics.stdev(dow_values[current_dow]) - if len(dow_values[current_dow]) > 1 - else 0 + statistics.stdev(dow_values[current_dow]) if len(dow_values[current_dow]) > 1 else 0 ) if dow_stdev > 0 and abs((current - dow_mean) / dow_stdev) < 1.5: return { "seasonal_anomaly": False, "seasonal_explanation": ( f"Value fits " - f"{['Mon','Tue','Wed','Thu','Fri','Sat','Sun'][current_dow]} pattern" + f"{['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'][current_dow]} pattern" ), } if context.get("is_promotional"): diff --git a/api.py b/api.py index 68f3098..7de7ffa 100644 --- a/api.py +++ b/api.py @@ -65,7 +65,9 @@ logger = logging.getLogger(__name__) # service, and event. setup_logging() bridges stdlib logging through # structlog so that any logger.info("event", k=v) becomes a JSON record. try: - from logging_config import setup_logging, get_logger as _get_logger + from logging_config import get_logger as _get_logger + from logging_config import setup_logging + setup_logging() logger = _get_logger(__name__) except ImportError: @@ -649,7 +651,7 @@ async def scrape(request: ScrapeRequest) -> dict[str, Any]: return response except PryError: raise - except Exception as e: # noqa: BLE001 + except Exception as e: raise ExternalServiceError(str(e)) from e @@ -934,7 +936,7 @@ async def _run_crawl_job(job_id: str, request: CrawlRequest) -> None: }, ) await queue.complete_job(job_id, {"pages": pages}) - except Exception as e: # noqa: BLE001 + except Exception as e: logger.exception("crawl_job_failed", extra={"job_id": job_id, "url": request.url}) await queue.fail_job(job_id, str(e)) @@ -976,7 +978,7 @@ async def parse_document(request: ParseRequest) -> dict[str, Any]: return {"success": True, "data": result} except PryError: raise - except Exception as e: # noqa: BLE001 + except Exception as e: raise ExternalServiceError(str(e)) from e @@ -3134,6 +3136,7 @@ async def list_schemas() -> dict[str, Any]: # ── Auth ── # (split into routers/auth.py on the api-router-split refactor) from routers.auth import router as auth_router + app.include_router(auth_router) # ── Review ── @@ -4051,6 +4054,7 @@ async def proxy_signup(provider: str = Body(...)) -> dict[str, Any]: url = pm.get_signup_link(provider) return {"success": True, "data": {"signup_url": url, "provider": provider}} + @app.get("/v1/proxy/referrals", tags=["Proxy"], summary="List proxy provider affiliate referrals") async def list_proxy_referrals() -> dict[str, Any]: """Return the curated proxy provider affiliate catalog (proxy_referrals.py). @@ -4074,7 +4078,9 @@ async def list_proxy_referrals() -> dict[str, Any]: } -@app.get("/v1/proxy/referrals/{tag}", tags=["Proxy"], summary="Get a single proxy provider referral") +@app.get( + "/v1/proxy/referrals/{tag}", tags=["Proxy"], summary="Get a single proxy provider referral" +) async def get_proxy_referral(tag: str) -> dict[str, Any]: """Return the affiliate info for a single proxy provider by tag.""" from proxy_manager import ProxyManager diff --git a/auth.py b/auth.py index c7fd516..721e52a 100644 --- a/auth.py +++ b/auth.py @@ -20,6 +20,7 @@ logger = logging.getLogger(__name__) # Try to import JWT library try: import jwt + _has_jwt = True except ImportError: _has_jwt = False @@ -30,8 +31,11 @@ except ImportError: # an unset JWT_SECRET cannot accidentally sign tokens in a way that survives restart. try: from secrets_backend import get_secret - JWT_SECRET = get_secret("jwt_secret") or os.getenv("PRY_JWT_SECRET") or ( - "ephemeral-" + secrets.token_hex(32) + + JWT_SECRET = ( + get_secret("jwt_secret") + or os.getenv("PRY_JWT_SECRET") + or ("ephemeral-" + secrets.token_hex(32)) ) except ImportError: JWT_SECRET = os.getenv("PRY_JWT_SECRET") or ("ephemeral-" + secrets.token_hex(32)) @@ -51,7 +55,8 @@ class AuthManager: self._api_keys: dict[str, dict[str, Any]] = {} def hash_password(self, password: str, salt: str | None = None) -> tuple[str, str]: - if salt is None: salt = secrets.token_hex(16) + if salt is None: + salt = secrets.token_hex(16) h = hashlib.pbkdf2_hmac("sha256", password.encode(), salt.encode(), 100000) return h.hex(), salt @@ -63,21 +68,31 @@ class AuthManager: user_id = secrets.token_hex(12) pwd_hash, salt = self.hash_password(password) user = { - "id": user_id, "email": email, "password_hash": pwd_hash, "salt": salt, - "role": role, "created_at": datetime.now(UTC).isoformat(), - "active": True, "api_keys": [], + "id": user_id, + "email": email, + "password_hash": pwd_hash, + "salt": salt, + "role": role, + "created_at": datetime.now(UTC).isoformat(), + "active": True, + "api_keys": [], } self._users[user_id] = user return user - def create_api_key(self, user_id: str, name: str = "default", rate_limit_rpm: int = DEFAULT_RATE_LIMIT_RPM) -> str: + def create_api_key( + self, user_id: str, name: str = "default", rate_limit_rpm: int = DEFAULT_RATE_LIMIT_RPM + ) -> str: key = "pry_" + secrets.token_urlsafe(API_KEY_LENGTH) key_hash = hashlib.sha256(key.encode()).hexdigest() api_key = { - "key_hash": key_hash, "user_id": user_id, "name": name, + "key_hash": key_hash, + "user_id": user_id, + "name": name, "rate_limit_rpm": rate_limit_rpm, "created_at": datetime.now(UTC).isoformat(), - "last_used": None, "use_count": 0, + "last_used": None, + "use_count": 0, } self._api_keys[key_hash] = api_key if user_id in self._users: @@ -95,7 +110,8 @@ class AuthManager: return api_key def check_rate_limit(self, api_key: str) -> tuple[bool, int]: - if not api_key: return True, 0 + if not api_key: + return True, 0 now = time.time() rl = self._rate_limits.setdefault(api_key, {"window_start": now, "count": 0}) if now - rl["window_start"] > 60: @@ -103,7 +119,11 @@ class AuthManager: rl["count"] = 0 rl["count"] += 1 api_key_data = self.verify_api_key(api_key) - limit = api_key_data.get("rate_limit_rpm", DEFAULT_RATE_LIMIT_RPM) if api_key_data else DEFAULT_RATE_LIMIT_RPM + limit = ( + api_key_data.get("rate_limit_rpm", DEFAULT_RATE_LIMIT_RPM) + if api_key_data + else DEFAULT_RATE_LIMIT_RPM + ) remaining = max(0, limit - rl["count"]) return rl["count"] <= limit, remaining @@ -112,15 +132,21 @@ class AuthManager: # Fallback: simple base64 token (NOT for production) payload = {"sub": user_id, "role": role, "exp": time.time() + JWT_EXPIRY_HOURS * 3600} return "pry_jwt." + base64_encode(json.dumps(payload)) - payload = {"sub": user_id, "role": role, "exp": datetime.now(UTC) + timedelta(hours=JWT_EXPIRY_HOURS)} + payload = { + "sub": user_id, + "role": role, + "exp": datetime.now(UTC) + timedelta(hours=JWT_EXPIRY_HOURS), + } return jwt.encode(payload, JWT_SECRET, algorithm=JWT_ALGORITHM) def verify_jwt(self, token: str) -> dict[str, Any] | None: if not _has_jwt: try: - if not token.startswith("pry_jwt."): return None + if not token.startswith("pry_jwt."): + return None payload = json.loads(base64_decode(token[8:])) - if payload.get("exp", 0) < time.time(): return None + if payload.get("exp", 0) < time.time(): + return None return payload except (json.JSONDecodeError, ValueError): return None @@ -132,11 +158,14 @@ class AuthManager: def base64_encode(s: str) -> str: import base64 + return base64.urlsafe_b64encode(s.encode()).decode().rstrip("=") def base64_decode(s: str) -> str: import base64 + padding = 4 - len(s) % 4 - if padding != 4: s += "=" * padding + if padding != 4: + s += "=" * padding return base64.urlsafe_b64decode(s.encode()).decode() diff --git a/auth_connector.py b/auth_connector.py index 1bb24e9..8640092 100644 --- a/auth_connector.py +++ b/auth_connector.py @@ -1,15 +1,12 @@ -import httpx + """Pry — Enterprise SSO / Auth Connector System. Credential vault, session persistence, SSO flow automation, CAPTCHA integration.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: BSL-1.1 # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — Stealth / Anti-Detection Module # Licensed under Business Source License 1.1 — see LICENSE-BSL-STEALTH. # Change Date: 2029-01-01 (converts to MIT). - import asyncio import json import logging @@ -20,6 +17,10 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any, Literal, cast +import httpx + +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) VAULT_DIR = PRY_DATA_DIR / "vault" @@ -66,7 +67,11 @@ def store_credential( path.write_text(json.dumps(entry, indent=2)) logger.info( "credential_stored", - extra={"credential_id": credential_id, "credential_name": name, "type": credential_type}, + extra={ + "credential_id": credential_id, + "credential_name": name, + "type": credential_type, + }, ) return {"success": True, "credential_id": credential_id, "credential": entry} except OSError as e: diff --git a/automator.py b/automator.py index 964067b..22d9ab2 100644 --- a/automator.py +++ b/automator.py @@ -14,6 +14,7 @@ all user inputs validated before passing to Playwright. import asyncio import base64 +import contextlib import json import os import time @@ -291,15 +292,11 @@ class PryAutomator: async with self._lock: if session_id in self.sessions: session = self.sessions.pop(session_id) - try: + with contextlib.suppress(OSError): await session.browser.close() - except OSError: - pass if os.path.exists(session.cookies_file): - try: + with contextlib.suppress(OSError): os.remove(session.cookies_file) - except OSError: - pass def list_sessions(self) -> list[dict]: return [ @@ -317,9 +314,5 @@ class PryAutomator: stale = [sid for sid, s in self.sessions.items() if now - s.last_used > SESSION_MAX_AGE] for sid in stale: session = self.sessions.pop(sid) - try: + with contextlib.suppress(OSError): await session.browser.close() - except OSError: - pass - - diff --git a/behavioral_biometrics.py b/behavioral_biometrics.py index eb67f3f..2c485f7 100644 --- a/behavioral_biometrics.py +++ b/behavioral_biometrics.py @@ -56,14 +56,14 @@ class HumanBehaviorSimulator: x = ( (1 - t) ** 3 * start[0] + 3 * (1 - t) ** 2 * t * ctrl1[0] - + 3 * (1 - t) * t ** 2 * ctrl2[0] - + t ** 3 * end[0] + + 3 * (1 - t) * t**2 * ctrl2[0] + + t**3 * end[0] ) y = ( (1 - t) ** 3 * start[1] + 3 * (1 - t) ** 2 * t * ctrl1[1] - + 3 * (1 - t) * t ** 2 * ctrl2[1] - + t ** 3 * end[1] + + 3 * (1 - t) * t**2 * ctrl2[1] + + t**3 * end[1] ) # Speed: slow at start/end, fast in middle speed_mod = math.sin(t * math.pi) * 0.5 + 0.5 # 0 at endpoints, 1 in middle @@ -91,9 +91,7 @@ class HumanBehaviorSimulator: micro_pauses = max(0, words // 20) * random.uniform(0.5, 2.0) return round(seconds * variance + micro_pauses, 2) - def scroll_pattern( - self, page_height: int, viewport_height: int = 800 - ) -> list[dict[str, Any]]: + def scroll_pattern(self, page_height: int, viewport_height: int = 800) -> list[dict[str, Any]]: """Generate realistic scroll pattern for a page. Humans don't scroll linearly — they scroll, pause, scroll back, etc. @@ -129,9 +127,7 @@ class HumanBehaviorSimulator: current_y = min(page_height, current_y + scroll_amount) # Pause longer on certain content (images, headings) pause = ( - random.uniform(1.0, 4.0) - if random.random() < 0.2 - else random.uniform(0.2, 1.0) + random.uniform(1.0, 4.0) if random.random() < 0.2 else random.uniform(0.2, 1.0) ) patterns.append( { @@ -141,9 +137,7 @@ class HumanBehaviorSimulator: } ) # Final scroll to bottom - patterns.append( - {"y": page_height, "speed": "fast", "pause_after": 0.5} - ) + patterns.append({"y": page_height, "speed": "fast", "pause_after": 0.5}) return patterns def typing_pattern(self, text: str) -> list[dict[str, Any]]: @@ -154,8 +148,22 @@ class HumanBehaviorSimulator: """ timings: list[dict[str, Any]] = [] common_words = { - "the", "a", "an", "is", "are", "was", "and", "or", "but", - "in", "on", "at", "to", "for", "of", "with", + "the", + "a", + "an", + "is", + "are", + "was", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "of", + "with", } words = text.split(" ") for i, word in enumerate(words): diff --git a/cache.py b/cache.py index 7007dbf..3971193 100644 --- a/cache.py +++ b/cache.py @@ -19,7 +19,7 @@ class ResponseCache: Default TTL: 300 seconds (5 min) for pages, 3600 (1 hr) for API calls. """ - def __init__(self, capacity: int = 500, redis_url: str = None): + def __init__(self, capacity: int = 500, redis_url: str | None = None): self.capacity = capacity self._cache: OrderedDict = OrderedDict() self._redis = None diff --git a/camoufox_integration.py b/camoufox_integration.py index 0bf77c2..6b15dfc 100644 --- a/camoufox_integration.py +++ b/camoufox_integration.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) try: from camoufox import AsyncCamoufox from camoufox.utils import DefaultAddons + _has_camoufox = True except ImportError: _has_camoufox = False @@ -31,28 +32,42 @@ class CamoufoxBrowser: DEFAULT_CONFIGS = { "chrome_windows": { - "headless": True, "os": "windows", "browser": "chrome", - "screen": (1920, 1080), "window": (1920, 1080), + "headless": True, + "os": "windows", + "browser": "chrome", + "screen": (1920, 1080), + "window": (1920, 1080), }, "firefox_windows": { - "headless": True, "os": "windows", "browser": "firefox", - "screen": (1920, 1080), "window": (1920, 1080), + "headless": True, + "os": "windows", + "browser": "firefox", + "screen": (1920, 1080), + "window": (1920, 1080), }, "chrome_mac": { - "headless": True, "os": "macos", "browser": "chrome", - "screen": (2560, 1600), "window": (1440, 900), + "headless": True, + "os": "macos", + "browser": "chrome", + "screen": (2560, 1600), + "window": (1440, 900), }, "firefox_linux": { - "headless": True, "os": "linux", "browser": "firefox", - "screen": (1920, 1080), "window": (1920, 1080), + "headless": True, + "os": "linux", + "browser": "firefox", + "screen": (1920, 1080), + "window": (1920, 1080), }, } def __init__(self, default_profile: str = "chrome_windows"): self.default_profile = default_profile if not _has_camoufox: - logger.warning("camoufox_not_available", - extra={"hint": "pip install camoufox && python -m camoufox fetch"}) + logger.warning( + "camoufox_not_available", + extra={"hint": "pip install camoufox && python -m camoufox fetch"}, + ) async def fetch( self, @@ -74,10 +89,16 @@ class CamoufoxBrowser: cookies: Cookies to set before navigation """ if not _has_camoufox: - return {"success": False, "error": "camoufox not installed. Run: pip install camoufox && python -m camoufox fetch"} + return { + "success": False, + "error": "camoufox not installed. Run: pip install camoufox && python -m camoufox fetch", + } profile_name = profile or self.default_profile - config = dict(self.DEFAULT_CONFIGS.get(profile_name, self.DEFAULT_CONFIGS["chrome_windows"])) - if proxy: config["proxy"] = proxy + config = dict( + self.DEFAULT_CONFIGS.get(profile_name, self.DEFAULT_CONFIGS["chrome_windows"]) + ) + if proxy: + config["proxy"] = proxy try: start = time.time() async with AsyncCamoufox(**config) as browser: @@ -98,10 +119,15 @@ class CamoufoxBrowser: elapsed = time.time() - start cookies_received = await page.context.cookies() return { - "success": True, "url": url, "title": title, - "content": content, "raw_html": content, - "status_code": 200, "elapsed": round(elapsed, 2), - "profile": profile_name, "cookies": cookies_received, + "success": True, + "url": url, + "title": title, + "content": content, + "raw_html": content, + "status_code": 200, + "elapsed": round(elapsed, 2), + "profile": profile_name, + "cookies": cookies_received, } except Exception as e: # noqa: BLE001 return {"success": False, "error": str(e)[:300], "url": url} @@ -120,8 +146,10 @@ class CamoufoxBrowser: if human_behavior: try: from camoufox import AsyncCamoufox + config = dict(self.DEFAULT_CONFIGS.get(self.default_profile, {})) - if proxy: config["proxy"] = proxy + if proxy: + config["proxy"] = proxy async with AsyncCamoufox(**config) as browser: page = await browser.new_page() await page.goto(url, wait_until="domcontentloaded") diff --git a/captcha_solver.py b/captcha_solver.py index b2622da..f19747e 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -21,15 +21,25 @@ logger = logging.getLogger(__name__) class CaptchaSolver: """Unified CAPTCHA solver with 6+ providers and automatic fallback chain.""" - PROVIDER_PRIORITY: ClassVar[list[str]] = ["capsolver", "2captcha", "anti_captcha", "capmonster", "deathbycaptcha", "nextcaptcha"] + PROVIDER_PRIORITY: ClassVar[list[str]] = [ + "capsolver", + "2captcha", + "anti_captcha", + "capmonster", + "deathbycaptcha", + "nextcaptcha", + ] def __init__(self, api_keys: dict[str, str] | None = None): self.api_keys = api_keys or {} self.provider_stats: dict[str, dict[str, Any]] = { - p: {"success": 0, "failed": 0, "avg_time": 0, "last_error": ""} for p in self.PROVIDER_PRIORITY + p: {"success": 0, "failed": 0, "avg_time": 0, "last_error": ""} + for p in self.PROVIDER_PRIORITY } - async def solve_recaptcha_v2(self, site_key: str, page_url: str, provider: str = "") -> dict[str, Any]: + async def solve_recaptcha_v2( + self, site_key: str, page_url: str, provider: str = "" + ) -> dict[str, Any]: """Solve reCAPTCHA v2.""" providers = [provider] if provider else self.PROVIDER_PRIORITY for prov in providers: @@ -41,24 +51,41 @@ class CaptchaSolver: result = await self._solve_with(prov, "ReCaptchaV2Task", site_key, page_url, key) elapsed = time.time() - start self._record(prov, True, elapsed) - return {"success": True, "provider": prov, "token": result, "elapsed": round(elapsed, 2)} + return { + "success": True, + "provider": prov, + "token": result, + "elapsed": round(elapsed, 2), + } except Exception as e: # noqa: BLE001 self._record(prov, False, 0, str(e)) - logger.warning("captcha_provider_failed", extra={"provider": prov, "error": str(e)[:80]}) + logger.warning( + "captcha_provider_failed", extra={"provider": prov, "error": str(e)[:80]} + ) return {"success": False, "error": "All CAPTCHA providers failed"} - async def solve_recaptcha_v3(self, site_key: str, page_url: str, action: str = "verify", min_score: float = 0.3) -> dict[str, Any]: + async def solve_recaptcha_v3( + self, site_key: str, page_url: str, action: str = "verify", min_score: float = 0.3 + ) -> dict[str, Any]: """Solve reCAPTCHA v3 (invisible, returns score).""" for prov in self.PROVIDER_PRIORITY: key = self.api_keys.get(prov) or self.api_keys.get(prov + "_api_key", "") if not key: continue try: - result = await self._solve_with(prov, "ReCaptchaV3Task", site_key, page_url, key, - extra={"action": action, "minScore": min_score}) + result = await self._solve_with( + prov, + "ReCaptchaV3Task", + site_key, + page_url, + key, + extra={"action": action, "minScore": min_score}, + ) return {"success": True, "provider": prov, "token": result} except Exception as e: # noqa: BLE001 - logger.warning("recaptcha_v3_failed", extra={"provider": prov, "error": str(e)[:80]}) + logger.warning( + "recaptcha_v3_failed", extra={"provider": prov, "error": str(e)[:80]} + ) return {"success": False, "error": "All reCAPTCHA v3 providers failed"} async def solve_turnstile(self, site_key: str, page_url: str) -> dict[str, Any]: @@ -97,10 +124,20 @@ class CaptchaSolver: result = await self._solve_image_with(prov, image_base64, key, case_sensitive) return {"success": True, "provider": prov, "text": result} except Exception as e: # noqa: BLE001 - logger.warning("image_captcha_failed", extra={"provider": prov, "error": str(e)[:80]}) + logger.warning( + "image_captcha_failed", extra={"provider": prov, "error": str(e)[:80]} + ) return {"success": False, "error": "All image CAPTCHA providers failed"} - async def _solve_with(self, provider: str, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _solve_with( + self, + provider: str, + task_type: str, + site_key: str, + page_url: str, + api_key: str, + extra: dict | None = None, + ) -> str: if provider == "capsolver": return await self._capsolver(task_type, site_key, page_url, api_key, extra) elif provider == "2captcha": @@ -115,22 +152,30 @@ class CaptchaSolver: return await self._nextcaptcha(task_type, site_key, page_url, api_key, extra) raise ValueError(f"Unknown provider: {provider}") - async def _capsolver(self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _capsolver( + self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None + ) -> str: from client import get_client + client = await get_client() task: dict[str, Any] = {"type": task_type, "websiteKey": site_key, "websiteURL": page_url} if extra: task.update(extra) - resp = await client.post("https://api.capsolver.com/createTask", - json={"clientKey": api_key, "task": task}, timeout=30) + resp = await client.post( + "https://api.capsolver.com/createTask", + json={"clientKey": api_key, "task": task}, + timeout=30, + ) data = resp.json() task_id = data.get("taskId") if not task_id: raise Exception(data.get("errorDescription", "Capsolver error")) for _ in range(60): await asyncio.sleep(2) - r = await client.post("https://api.capsolver.com/getTaskResult", - json={"clientKey": api_key, "taskId": task_id}) + r = await client.post( + "https://api.capsolver.com/getTaskResult", + json={"clientKey": api_key, "taskId": task_id}, + ) rd = r.json() if rd.get("status") == "ready": return rd["solution"].get("gRecaptchaResponse") or rd["solution"].get("token", "") @@ -138,13 +183,25 @@ class CaptchaSolver: raise Exception("Capsolver failed") raise Exception("Capsolver timed out") - async def _two_captcha(self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _two_captcha( + self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None + ) -> str: from client import get_client + client = await get_client() - method = {"ReCaptchaV2Task": "userrecaptcha", "ReCaptchaV3Task": "recaptcha_v3", - "HCaptchaTask": "hcaptcha", "TurnstileTask": "turnstile"}.get(task_type, "userrecaptcha") - params: dict[str, Any] = {"key": api_key, "method": method, "googlekey": site_key, "pageurl": page_url, - "json": 1} + method = { + "ReCaptchaV2Task": "userrecaptcha", + "ReCaptchaV3Task": "recaptcha_v3", + "HCaptchaTask": "hcaptcha", + "TurnstileTask": "turnstile", + }.get(task_type, "userrecaptcha") + params: dict[str, Any] = { + "key": api_key, + "method": method, + "googlekey": site_key, + "pageurl": page_url, + "json": 1, + } if extra: params.update(extra) resp = await client.post("https://2captcha.com/in.php", data=params, timeout=30) @@ -154,7 +211,9 @@ class CaptchaSolver: request_id = data["request"] for _ in range(60): await asyncio.sleep(3) - r = await client.get(f"https://2captcha.com/res.php?key={api_key}&action=get&id={request_id}&json=1") + r = await client.get( + f"https://2captcha.com/res.php?key={api_key}&action=get&id={request_id}&json=1" + ) rd = r.json() if rd.get("status") == 1: return rd["request"] @@ -162,24 +221,40 @@ class CaptchaSolver: raise Exception(f"2captcha: {rd['request']}") raise Exception("2captcha timed out") - async def _anti_captcha(self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _anti_captcha( + self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None + ) -> str: from client import get_client + client = await get_client() - type_map = {"ReCaptchaV2Task": "NoCaptchaTaskProxyless", "ReCaptchaV3Task": "RecaptchaV3TaskProxyless", - "HCaptchaTask": "HCaptchaTaskProxyless", "TurnstileTask": "TurnstileTaskProxyless"} - task = {"type": type_map.get(task_type, "NoCaptchaTaskProxyless"), "websiteURL": page_url, "websiteKey": site_key} + type_map = { + "ReCaptchaV2Task": "NoCaptchaTaskProxyless", + "ReCaptchaV3Task": "RecaptchaV3TaskProxyless", + "HCaptchaTask": "HCaptchaTaskProxyless", + "TurnstileTask": "TurnstileTaskProxyless", + } + task = { + "type": type_map.get(task_type, "NoCaptchaTaskProxyless"), + "websiteURL": page_url, + "websiteKey": site_key, + } if extra: task.update(extra) - resp = await client.post("https://api.anti-captcha.com/createTask", - json={"clientKey": api_key, "task": task}, timeout=30) + resp = await client.post( + "https://api.anti-captcha.com/createTask", + json={"clientKey": api_key, "task": task}, + timeout=30, + ) data = resp.json() if data.get("errorId") != 0: raise Exception(data.get("errorDescription", "Anti-Captcha error")) task_id = data["taskId"] for _ in range(60): await asyncio.sleep(2) - r = await client.post("https://api.anti-captcha.com/getTaskResult", - json={"clientKey": api_key, "taskId": task_id}) + r = await client.post( + "https://api.anti-captcha.com/getTaskResult", + json={"clientKey": api_key, "taskId": task_id}, + ) rd = r.json() if rd.get("status") == "ready": return rd["solution"].get("gRecaptchaResponse", "") @@ -187,17 +262,25 @@ class CaptchaSolver: raise Exception(rd.get("errorDescription", "")) raise Exception("Anti-Captcha timed out") - async def _capmonster(self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _capmonster( + self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None + ) -> str: """CapMonster Cloud — self-hosted option. Can run locally with 0 per-solve fees.""" - return await self._anti_captcha(task_type, site_key, page_url, api_key, extra) # Same API as Anti-Captcha + return await self._anti_captcha( + task_type, site_key, page_url, api_key, extra + ) # Same API as Anti-Captcha - async def _deathbycaptcha(self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _deathbycaptcha( + self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None + ) -> str: """Solve via DeathByCaptcha API.""" from client import get_client client = await get_client() user, pw = api_key.split(":", 1) if ":" in api_key else (api_key, "") - method = {"ReCaptchaV2Task": "userrecaptcha", "HCaptchaTask": "hcaptcha"}.get(task_type, "userrecaptcha") + method = {"ReCaptchaV2Task": "userrecaptcha", "HCaptchaTask": "hcaptcha"}.get( + task_type, "userrecaptcha" + ) payload = { "username": user, "password": pw, @@ -217,7 +300,9 @@ class CaptchaSolver: return rd["text"] raise Exception("DBC timed out") - async def _nextcaptcha(self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None) -> str: + async def _nextcaptcha( + self, task_type: str, site_key: str, page_url: str, api_key: str, extra: dict | None = None + ) -> str: """Solve via NextCaptcha API.""" from client import get_client @@ -256,7 +341,9 @@ class CaptchaSolver: raise Exception(rd.get("errorDescription", "")) raise Exception("NextCaptcha timed out") - async def _solve_image_with(self, provider: str, image_base64: str, api_key: str, case_sensitive: bool) -> str: + async def _solve_image_with( + self, provider: str, image_base64: str, api_key: str, case_sensitive: bool + ) -> str: if provider == "capsolver": return await self._capsolver_image(image_base64, api_key, case_sensitive) elif provider == "2captcha": @@ -265,38 +352,60 @@ class CaptchaSolver: async def _capsolver_image(self, image_base64: str, api_key: str, case_sensitive: bool) -> str: from client import get_client + client = await get_client() - body = image_base64 if not image_base64.startswith("data:") else image_base64.split(",", 1)[1] - resp = await client.post("https://api.capsolver.com/createTask", - json={"clientKey": api_key, "task": {"type": "ImageToTextTask", "body": body[:100000]}}, - timeout=30) + body = ( + image_base64 if not image_base64.startswith("data:") else image_base64.split(",", 1)[1] + ) + resp = await client.post( + "https://api.capsolver.com/createTask", + json={"clientKey": api_key, "task": {"type": "ImageToTextTask", "body": body[:100000]}}, + timeout=30, + ) data = resp.json() task_id = data.get("taskId") if not task_id: raise Exception(data.get("errorDescription", "")) for _ in range(30): await asyncio.sleep(2) - r = await client.post("https://api.capsolver.com/getTaskResult", - json={"clientKey": api_key, "taskId": task_id}) + r = await client.post( + "https://api.capsolver.com/getTaskResult", + json={"clientKey": api_key, "taskId": task_id}, + ) rd = r.json() if rd.get("status") == "ready": return rd["solution"].get("text", "") raise Exception("Image solve timed out") - async def _two_captcha_image(self, image_base64: str, api_key: str, case_sensitive: bool) -> str: + async def _two_captcha_image( + self, image_base64: str, api_key: str, case_sensitive: bool + ) -> str: from client import get_client + client = await get_client() - body = image_base64 if not image_base64.startswith("data:") else image_base64.split(",", 1)[1] - resp = await client.post("https://2captcha.com/in.php", - data={"key": api_key, "method": "base64", "body": body, "json": 1, - "regsense": 1 if case_sensitive else 0}, timeout=30) + body = ( + image_base64 if not image_base64.startswith("data:") else image_base64.split(",", 1)[1] + ) + resp = await client.post( + "https://2captcha.com/in.php", + data={ + "key": api_key, + "method": "base64", + "body": body, + "json": 1, + "regsense": 1 if case_sensitive else 0, + }, + timeout=30, + ) data = resp.json() if data.get("status") != 1: raise Exception(data.get("request", "")) request_id = data["request"] for _ in range(30): await asyncio.sleep(3) - r = await client.get(f"https://2captcha.com/res.php?key={api_key}&action=get&id={request_id}&json=1") + r = await client.get( + f"https://2captcha.com/res.php?key={api_key}&action=get&id={request_id}&json=1" + ) rd = r.json() if rd.get("status") == 1: return rd["request"] @@ -306,10 +415,15 @@ class CaptchaSolver: stats = self.provider_stats[provider] if success: stats["success"] += 1 - stats["avg_time"] = (stats["avg_time"] * (stats["success"] + stats["failed"] - 1) + elapsed) / (stats["success"] + stats["failed"]) + stats["avg_time"] = ( + stats["avg_time"] * (stats["success"] + stats["failed"] - 1) + elapsed + ) / (stats["success"] + stats["failed"]) else: stats["failed"] += 1 stats["last_error"] = error[:100] def get_stats(self) -> dict[str, Any]: - return {"providers": self.provider_stats, "healthy_providers": sum(1 for p in self.provider_stats.values() if p["failed"] < 3)} + return { + "providers": self.provider_stats, + "healthy_providers": sum(1 for p in self.provider_stats.values() if p["failed"] < 3), + } diff --git a/cli.py b/cli.py index 25a1cfe..a44943d 100755 --- a/cli.py +++ b/cli.py @@ -47,7 +47,7 @@ def _api(): return os.getenv("PRY_URL", API_DEFAULT) -def _req(method: str, path: str, data: dict = None, timeout=30): +def _req(method: str, path: str, data: dict | None = None, timeout=30): import httpx url = f"{_api()}{path}" @@ -101,7 +101,7 @@ def cmd_open(url, output_json=False, schema_path=None, timeout=30): if schema_path: with open(schema_path) as f: payload["jsonSchema"] = json.load(f) - s = _spinner(f"Prying open {url[:50]}") + _spinner(f"Prying open {url[:50]}") data = _req("POST", "/v1/scrape", payload, timeout + 10) print(f"{GREEN}✓{NC} Pry opened {url}\n", end="") if output_json or schema_path: @@ -113,7 +113,7 @@ def cmd_open(url, output_json=False, schema_path=None, timeout=30): def cmd_watch(url, webhook="", interval=3600): """Register a page for change monitoring.""" - s = _spinner(f"Registering {url[:50]} for monitoring") + _spinner(f"Registering {url[:50]} for monitoring") data = _req("POST", "/v1/watch", {"url": url, "webhook": webhook, "interval": interval}, 45) if data.get("success"): status = data.get("data", {}).get("status", "registered") @@ -126,7 +126,7 @@ def cmd_watch(url, webhook="", interval=3600): def cmd_crawl(url, max_pages=10, output=None, timeout=120): """Crawl multiple pages from a starting URL.""" - s = _spinner(f"Crawling {url[:50]} (up to {max_pages} pages)") + _spinner(f"Crawling {url[:50]} (up to {max_pages} pages)") data = _req("POST", "/v1/crawl", {"url": url, "maxPages": max_pages}, timeout) pages = data.get("data", {}).get("pages", []) print(f"{GREEN}✓{NC} Crawled {len(pages)} page(s) from {url}") @@ -148,7 +148,7 @@ def cmd_batch(filepath, template_str="", timeout=30): print(f"{RED}✖ File not found: {filepath}{NC}") sys.exit(1) template = json.loads(template_str) if template_str else {"body": "body"} - s = _spinner(f"Processing {filepath}") + _spinner(f"Processing {filepath}") data = _req( "POST", "/v1/batch-file", @@ -168,7 +168,7 @@ def cmd_batch(filepath, template_str="", timeout=30): def cmd_parse(url, timeout=60): """Parse a document to text.""" - s = _spinner(f"Parsing {url[:50]}") + _spinner(f"Parsing {url[:50]}") data = _req("POST", "/v1/parse", {"url": url, "timeout": timeout}, timeout + 10) text = data.get("data", {}).get("text", "") fmt = data.get("data", {}).get("format", "unknown") @@ -179,7 +179,7 @@ def cmd_parse(url, timeout=60): def cmd_screenshot(url, output=None, timeout=30): """Take a screenshot.""" - s = _spinner(f"Capturing {url[:50]}") + _spinner(f"Capturing {url[:50]}") data = _req("POST", "/v1/screenshot", {"url": url}, timeout + 10) b64 = data.get("data", {}).get("screenshot", "") if not b64: @@ -199,7 +199,7 @@ def cmd_run(pryfile_path="pry.yml"): print(f"{RED}✖ No {pryfile_path} found{NC}") print(f" Create one: {CYAN}pry open https://example.com{NC}") sys.exit(1) - s = _spinner(f"Running jobs from {pryfile_path}") + _spinner(f"Running jobs from {pryfile_path}") data = _req("POST", "/v1/run", {"path": pryfile_path}, 120) jobs = data.get("data", {}).get("jobs", []) print(f"{GREEN}✓{NC} Executed {len(jobs)} job(s)") @@ -376,10 +376,12 @@ if click is not None: def mcp_serve() -> None: """Start the MCP server (stdio transport).""" import sys + sys.path.insert(0, ".") import asyncio from mcp_production import main + asyncio.run(main()) @mcp.command(name="info") @@ -455,8 +457,7 @@ if click is not None: creds["proxy_url"] = proxy_url if not creds: click.echo( - "Need at least one credential " - "(--username, --password, --api-key, or --proxy-url)" + "Need at least one credential (--username, --password, --api-key, or --proxy-url)" ) return result = pm.select_provider(provider, creds) diff --git a/commerce_sync.py b/commerce_sync.py index 9a9e9a1..9875536 100644 --- a/commerce_sync.py +++ b/commerce_sync.py @@ -1,19 +1,18 @@ -import httpx + """Pry — Commerce Platform Sync Engine. Unified interface for WooCommerce, Shopify, and generic API sync.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import logging -import os -from pathlib import Path from typing import Any +import httpx + +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) # ── Record sync operation ── diff --git a/compliance.py b/compliance.py index 44f9e07..909334d 100644 --- a/compliance.py +++ b/compliance.py @@ -354,6 +354,7 @@ async def run_compliance_check(url: str) -> dict[str, Any]: if tos_result.get("confidence") == "low" or not tos_text: try: from llm_features import llm_compliance_analyze + llm_input = tos_text if tos_text else html[:8000] llm_result = await llm_compliance_analyze(llm_input, url=url) if llm_result and llm_result.get("risk_level"): diff --git a/config.py b/config.py index 9e09c3c..f979598 100644 --- a/config.py +++ b/config.py @@ -11,11 +11,9 @@ All secrets come from gopass, never from .env files committed to git. # Licensed under MIT. See LICENSE. from __future__ import annotations -import os from pathlib import Path from typing import Literal -from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict diff --git a/cookie_warmer.py b/cookie_warmer.py index 93f8019..ea52cae 100644 --- a/cookie_warmer.py +++ b/cookie_warmer.py @@ -27,16 +27,29 @@ WARMER_DIR.mkdir(parents=True, exist_ok=True) # Realistic pages to "warm" cookies against (these are common referrers/browsing paths) WARMING_PAGES = { - "amazon": ["https://www.amazon.com/", "https://www.amazon.com/gp/help/customer/contact-us", - "https://www.amazon.com/privacy", "https://www.amazon.com/conditions-of-use"], - "ebay": ["https://www.ebay.com/", "https://www.ebay.com/help/home", - "https://www.ebay.com/myp/PurchaseHistory"], + "amazon": [ + "https://www.amazon.com/", + "https://www.amazon.com/gp/help/customer/contact-us", + "https://www.amazon.com/privacy", + "https://www.amazon.com/conditions-of-use", + ], + "ebay": [ + "https://www.ebay.com/", + "https://www.ebay.com/help/home", + "https://www.ebay.com/myp/PurchaseHistory", + ], "shopify": ["https://www.shopify.com/", "https://www.shopify.com/pricing"], - "twitter": ["https://twitter.com/", "https://twitter.com/explore", - "https://twitter.com/settings/account"], + "twitter": [ + "https://twitter.com/", + "https://twitter.com/explore", + "https://twitter.com/settings/account", + ], "linkedin": ["https://www.linkedin.com/", "https://www.linkedin.com/help/intro"], - "generic": ["https://www.google.com/", "https://en.wikipedia.org/wiki/Main_Page", - "https://github.com/"], + "generic": [ + "https://www.google.com/", + "https://en.wikipedia.org/wiki/Main_Page", + "https://github.com/", + ], } diff --git a/costing.py b/costing.py index fd0b9a5..2064052 100644 --- a/costing.py +++ b/costing.py @@ -1,21 +1,19 @@ """Pry — Cost Analytics Engine. Tracks usage costs, cache hit rates, projected burn, and smart scheduling.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging -import os from collections import defaultdict from datetime import UTC, datetime, timedelta -from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) COSTING_DIR = PRY_DATA_DIR / "costing" diff --git a/crm_sync.py b/crm_sync.py index 7f01b45..8254fe8 100644 --- a/crm_sync.py +++ b/crm_sync.py @@ -1,4 +1,4 @@ -import httpx + """Pry — Reverse ETL to CRM. Sync scraped data to Salesforce, HubSpot, Pipedrive, and Close.com.""" @@ -11,6 +11,8 @@ Sync scraped data to Salesforce, HubSpot, Pipedrive, and Close.com.""" import logging from typing import Any +import httpx + logger = logging.getLogger(__name__) diff --git a/db.py b/db.py index 9ef2b6d..9300412 100644 --- a/db.py +++ b/db.py @@ -50,6 +50,7 @@ the corresponding JSON store is migrated. Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper Licensed under MIT. See LICENSE. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -61,10 +62,11 @@ import json import logging import os import sqlite3 +from collections.abc import Iterator from contextlib import contextmanager from datetime import UTC, datetime from pathlib import Path -from typing import Any, Iterator +from typing import Any try: from sqlalchemy import ( @@ -88,7 +90,7 @@ try: except ImportError: # pragma: no cover _HAS_SA = False -from paths import PRY_DATA_DIR # noqa: E402 +from paths import PRY_DATA_DIR logger = logging.getLogger(__name__) @@ -98,10 +100,11 @@ DEFAULT_DB_PATH = PRY_DATA_DIR / "pry.db" _has_sqlalchemy = _HAS_SA -def get_db() -> "Engine": +def get_db() -> Engine: """Backward-compat alias for get_engine().""" return get_engine() + DATABASE_URL_ENV = "PRY_DATABASE_URL" DEFAULT_SQLITE_URL = f"sqlite:///{DEFAULT_DB_PATH}" @@ -114,11 +117,11 @@ def _resolve_database_url() -> str: return DEFAULT_SQLITE_URL -_engine: "Engine | None" = None -_SessionLocal: "sessionmaker[Session] | None" = None +_engine: Engine | None = None +_SessionLocal: sessionmaker[Session] | None = None -def get_engine() -> "Engine": +def get_engine() -> Engine: """Get the SQLAlchemy engine. Lazily creates one on first call.""" global _engine, _SessionLocal if not _HAS_SA: @@ -131,19 +134,23 @@ def get_engine() -> "Engine": _engine = create_engine(url, connect_args=connect_args, future=True, echo=False) # Enable foreign keys for SQLite (off by default) if url.startswith("sqlite"): + @event.listens_for(_engine, "connect") def _enable_sqlite_fk(dbapi_conn: sqlite3.Connection, _conn_record: Any) -> None: cur = dbapi_conn.cursor() cur.execute("PRAGMA foreign_keys=ON") cur.close() + _SessionLocal = sessionmaker(bind=_engine, autoflush=False, expire_on_commit=False) # Auto-create tables on first import (idempotent) Base.metadata.create_all(_engine) - logger.info("db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url}) + logger.info( + "db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url} + ) return _engine -def get_session() -> "Session": +def get_session() -> Session: """Get a new Session. Caller is responsible for closing it. Use `with db_session() as s:` or the `session_scope()` context manager @@ -156,7 +163,7 @@ def get_session() -> "Session": @contextmanager -def session_scope() -> Iterator["Session"]: +def session_scope() -> Iterator[Session]: """Context manager: yields a Session, commits on success, rolls back on error. Usage: @@ -178,6 +185,7 @@ def session_scope() -> Iterator["Session"]: # ── Model base ──────────────────────────────────────────────────── if _HAS_SA: + class Base(DeclarativeBase): pass @@ -282,7 +290,9 @@ if _HAS_SA: class MonitorRun(Base): __tablename__ = "monitor_runs" id = Column(Integer, primary_key=True) - monitor_id = Column(String(64), ForeignKey("monitors.monitor_id"), index=True, nullable=False) + monitor_id = Column( + String(64), ForeignKey("monitors.monitor_id"), index=True, nullable=False + ) ts = Column(DateTime, default=_now, index=True) status = Column(String(16), default="success") # success|changed|error diff = Column(JSON, default=dict) @@ -350,7 +360,9 @@ if _HAS_SA: class PipelineRun(Base): __tablename__ = "pipeline_runs" id = Column(Integer, primary_key=True) - pipeline_id = Column(String(64), ForeignKey("pipelines.pipeline_id"), index=True, nullable=False) + pipeline_id = Column( + String(64), ForeignKey("pipelines.pipeline_id"), index=True, nullable=False + ) ts = Column(DateTime, default=_now, index=True) status = Column(String(16), default="success") result = Column(JSON, default=dict) @@ -477,6 +489,7 @@ else: # pragma: no cover # ── Importer: JSON store -> SQL ─────────────────────────────────── + def _iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: if not path.exists(): return @@ -526,29 +539,33 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: # Quality n = 0 for rec in _iter_jsonl(data_dir / "quality" / "checks.jsonl"): - s.add(QualityCheck( - extraction_id=rec.get("extraction_id", rec.get("id", ""))[:64], - url=rec.get("url", ""), - completeness=rec.get("completeness", 0.0), - accuracy=rec.get("accuracy", 0.0), - freshness=rec.get("freshness", 0.0), - overall_score=rec.get("overall_score", 0.0), - risk_level=rec.get("risk_level", "unknown"), - issues=rec.get("issues", []), - data=rec, - )) + s.add( + QualityCheck( + extraction_id=rec.get("extraction_id", rec.get("id", ""))[:64], + url=rec.get("url", ""), + completeness=rec.get("completeness", 0.0), + accuracy=rec.get("accuracy", 0.0), + freshness=rec.get("freshness", 0.0), + overall_score=rec.get("overall_score", 0.0), + risk_level=rec.get("risk_level", "unknown"), + issues=rec.get("issues", []), + data=rec, + ) + ) n += 1 counts["quality"] = n # Intel (JSONL) n = 0 for rec in _iter_jsonl(data_dir / "intel" / "snapshots.jsonl"): - s.add(IntelSnapshot( - competitor_id=rec.get("competitor_id", "")[:64], - competitor_name=rec.get("competitor_name", "")[:256], - url=rec.get("url", ""), - fields=rec.get("fields", {}), - )) + s.add( + IntelSnapshot( + competitor_id=rec.get("competitor_id", "")[:64], + competitor_name=rec.get("competitor_name", "")[:256], + url=rec.get("url", ""), + fields=rec.get("fields", {}), + ) + ) n += 1 counts["intel"] = n @@ -556,15 +573,17 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: n = 0 for rec in _iter_json_files(data_dir / "monitors"): mid = rec.get("monitor_id") or rec.get("id", "") - s.add(Monitor( - monitor_id=mid[:64], - url=rec.get("url", ""), - monitor_name=rec.get("name", ""), # was 'name' in JSON - schedule_cron=rec.get("schedule_cron", ""), - check_type=rec.get("check_type", "content"), - active=rec.get("active", True), - webhook_url=rec.get("webhook_url", ""), - )) + s.add( + Monitor( + monitor_id=mid[:64], + url=rec.get("url", ""), + monitor_name=rec.get("name", ""), # was 'name' in JSON + schedule_cron=rec.get("schedule_cron", ""), + check_type=rec.get("check_type", "content"), + active=rec.get("active", True), + webhook_url=rec.get("webhook_url", ""), + ) + ) n += 1 counts["monitors"] = n @@ -572,14 +591,16 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: n = 0 for rec in _iter_json_files(data_dir / "accounts"): aid = rec.get("account_id") or rec.get("id", "") - s.add(Account( - account_id=aid[:64], - service=rec.get("service", "unknown")[:64], - username=rec.get("username", ""), - email=rec.get("email", ""), - status=rec.get("status", "active"), - used_count=rec.get("used_count", 0), - )) + s.add( + Account( + account_id=aid[:64], + service=rec.get("service", "unknown")[:64], + username=rec.get("username", ""), + email=rec.get("email", ""), + status=rec.get("status", "active"), + used_count=rec.get("used_count", 0), + ) + ) n += 1 counts["accounts"] = n @@ -587,20 +608,22 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: n = 0 for rec in _iter_json_files(data_dir / "actors"): aid = rec.get("actor_id") or rec.get("id", "") - s.add(Actor( - actor_id=aid[:64], - name=rec.get("name", ""), - description=rec.get("description", ""), - template_id=rec.get("template_id", ""), - code=rec.get("code", ""), - price_per_run=rec.get("price_per_run", 0.0), - visibility=rec.get("visibility", "private"), - schedule_cron=rec.get("schedule_cron", ""), - tags=rec.get("tags", []), - author=rec.get("author", ""), - run_count=rec.get("run_count", 0), - revenue_usd=rec.get("revenue_usd", 0.0), - )) + s.add( + Actor( + actor_id=aid[:64], + name=rec.get("name", ""), + description=rec.get("description", ""), + template_id=rec.get("template_id", ""), + code=rec.get("code", ""), + price_per_run=rec.get("price_per_run", 0.0), + visibility=rec.get("visibility", "private"), + schedule_cron=rec.get("schedule_cron", ""), + tags=rec.get("tags", []), + author=rec.get("author", ""), + run_count=rec.get("run_count", 0), + revenue_usd=rec.get("revenue_usd", 0.0), + ) + ) n += 1 counts["actors"] = n @@ -608,13 +631,15 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: n = 0 for rec in _iter_json_files(data_dir / "pipelines"): pid = rec.get("pipeline_id") or rec.get("id", "") - s.add(Pipeline( - pipeline_id=pid[:64], - pipeline_name=rec.get("name", ""), - steps=rec.get("steps", []), - schedule_cron=rec.get("schedule_cron", ""), - active=rec.get("active", True), - )) + s.add( + Pipeline( + pipeline_id=pid[:64], + pipeline_name=rec.get("name", ""), + steps=rec.get("steps", []), + schedule_cron=rec.get("schedule_cron", ""), + active=rec.get("active", True), + ) + ) n += 1 counts["pipelines"] = n @@ -622,13 +647,15 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: n = 0 for rec in _iter_json_files(data_dir / "sessions"): sid = rec.get("session_id") or rec.get("id", "") - s.add(BrowserSession( - session_id=sid[:64], - cookies=rec.get("cookies", []), - local_storage=rec.get("local_storage", {}), - user_agent=rec.get("user_agent", ""), - proxy=rec.get("proxy", ""), - )) + s.add( + BrowserSession( + session_id=sid[:64], + cookies=rec.get("cookies", []), + local_storage=rec.get("local_storage", {}), + user_agent=rec.get("user_agent", ""), + proxy=rec.get("proxy", ""), + ) + ) n += 1 counts["sessions"] = n @@ -636,15 +663,17 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]: n = 0 for rec in _iter_json_files(data_dir / "agency"): aid = rec.get("agency_id") or rec.get("id", "") - s.add(Agency( - agency_id=aid[:64], - agency_name=rec.get("name", ""), - owner_email=rec.get("owner_email", ""), - custom_domain=rec.get("custom_domain", ""), - brand_color=rec.get("brand_color", "#f59e0b"), - logo_url=rec.get("logo_url", ""), - quota=rec.get("quota", 10000), - )) + s.add( + Agency( + agency_id=aid[:64], + agency_name=rec.get("name", ""), + owner_email=rec.get("owner_email", ""), + custom_domain=rec.get("custom_domain", ""), + brand_color=rec.get("brand_color", "#f59e0b"), + logo_url=rec.get("logo_url", ""), + quota=rec.get("quota", 10000), + ) + ) n += 1 counts["agency"] = n @@ -666,7 +695,9 @@ def db_health() -> dict[str, Any]: sqlite_version = None return { "available": True, - "url": _resolve_database_url().split("@")[-1] if "@" in _resolve_database_url() else _resolve_database_url(), + "url": _resolve_database_url().split("@")[-1] + if "@" in _resolve_database_url() + else _resolve_database_url(), "sqlite_version": sqlite_version, } except Exception as e: diff --git a/dedup.py b/dedup.py index ce3308d..0710d95 100644 --- a/dedup.py +++ b/dedup.py @@ -118,9 +118,12 @@ class Deduplicator: "hamming_distance": distance, "changed": sim < self.threshold, "change_severity": ( - "high" if distance > 20 - else "medium" if distance > 10 - else "low" if distance > 3 + "high" + if distance > 20 + else "medium" + if distance > 10 + else "low" + if distance > 3 else "minimal" ), } diff --git a/email_scraper.py b/email_scraper.py index 095ac5f..b2961b4 100644 --- a/email_scraper.py +++ b/email_scraper.py @@ -1,4 +1,4 @@ -import httpx + """Pry — Email Inbox Scraping. Connect Gmail/Outlook, extract structured data from emails.""" @@ -14,6 +14,8 @@ import re from datetime import UTC, datetime, timedelta from typing import Any +import httpx + logger = logging.getLogger(__name__) # ── Email Data Extraction Patterns ── diff --git a/enrichment.py b/enrichment.py index 5ee8433..14441cb 100644 --- a/enrichment.py +++ b/enrichment.py @@ -1,4 +1,4 @@ -import httpx + """Pry — Data Enrichment Pipeline. Enrich scraped data with company info, social profiles, tech stack detection.""" @@ -12,6 +12,8 @@ import logging import re from typing import Any +import httpx + logger = logging.getLogger(__name__) diff --git a/extraction.py b/extraction.py index 4e94fce..b3d5064 100644 --- a/extraction.py +++ b/extraction.py @@ -1,4 +1,4 @@ -import httpx + """Pry — structured extraction strategies. CSS/XPath-based extraction (no LLM needed) + chunking strategies for LLM extraction.""" @@ -14,6 +14,7 @@ import re from collections.abc import Sequence from typing import Any +import httpx from lxml import html as lxml_html logger = logging.getLogger(__name__) diff --git a/freshness.py b/freshness.py index 4489e79..3394633 100644 --- a/freshness.py +++ b/freshness.py @@ -1,23 +1,22 @@ -import httpx + """Pry — Adaptive Freshness Scheduling. Conditional scraping, content fingerprinting, staleness dashboard, adaptive frequency.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import hashlib import json import logging -import os from contextlib import suppress from datetime import UTC, datetime -from pathlib import Path from typing import Any +import httpx + +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) FRESHNESS_DIR = PRY_DATA_DIR / "freshness" diff --git a/gdpr.py b/gdpr.py index 2dc47f2..8b7c424 100644 --- a/gdpr.py +++ b/gdpr.py @@ -1,13 +1,11 @@ """Pry — GDPR Compliance Portal. Data deletion API, consent management, retention policies, audit log.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import hashlib import json import logging @@ -18,6 +16,8 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) GDPR_DIR = PRY_DATA_DIR / "gdpr" diff --git a/gdpr_real.py b/gdpr_real.py index b5b3b49..bab18aa 100644 --- a/gdpr_real.py +++ b/gdpr_real.py @@ -16,15 +16,31 @@ from pathlib import Path from typing import Any from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) GDPR_DIR = PRY_DATA_DIR / "gdpr_real" GDPR_DIR.mkdir(parents=True, exist_ok=True) DATA_RESIDENCES = [ - "quality/", "reviews/", "intel/", "costing/", "freshness/", "structure/", "seo/", - "monitors/", "vault/", "accounts/", "reports/", "training/", "pipelines/", - "agency/", "compliance/", "caching/", "stealth_scripts/", "jobs/", + "quality/", + "reviews/", + "intel/", + "costing/", + "freshness/", + "structure/", + "seo/", + "monitors/", + "vault/", + "accounts/", + "reports/", + "training/", + "pipelines/", + "agency/", + "compliance/", + "caching/", + "stealth_scripts/", + "jobs/", ] @@ -77,13 +93,13 @@ class GDPRService: continue if subject_id in content or subject_lower in content.lower(): data_found["data_categories"].append(str(subdir)) - data_found["records"].setdefault(str(subdir), []).append({ - "file": str(f.relative_to(pry_dir)), - "size": f.stat().st_size, - "modified": datetime.fromtimestamp( - f.stat().st_mtime, UTC - ).isoformat(), - }) + data_found["records"].setdefault(str(subdir), []).append( + { + "file": str(f.relative_to(pry_dir)), + "size": f.stat().st_size, + "modified": datetime.fromtimestamp(f.stat().st_mtime, UTC).isoformat(), + } + ) data_found["total_records"] += 1 return data_found @@ -130,9 +146,7 @@ class GDPRService: s.query(QualityCheckRecord).filter( QualityCheckRecord.url.like(f"%{subject_id}%") ).delete() - s.query(ApiKey).filter( - ApiKey.name.like(f"%{subject_id}%") - ).delete() + s.query(ApiKey).filter(ApiKey.name.like(f"%{subject_id}%")).delete() except OSError: pass @@ -197,9 +211,7 @@ class GDPRService: "files_count": access_data["total_records"], } - def get_audit_log( - self, days_back: int = 30, subject_id: str = "" - ) -> list[dict[str, Any]]: + def get_audit_log(self, days_back: int = 30, subject_id: str = "") -> list[dict[str, Any]]: """Get audit log entries.""" entries: list[dict[str, Any]] = [] if not self._audit_log_path.exists(): diff --git a/graphql_discovery.py b/graphql_discovery.py index d707cba..d0c793e 100644 --- a/graphql_discovery.py +++ b/graphql_discovery.py @@ -1,4 +1,4 @@ -import httpx + """Pry — GraphQL Auto-Discovery. Detects GraphQL endpoints, runs introspection queries, generates optimized queries. Many modern sites (Shopify, GitHub, Twitter/X, etc.) have GraphQL APIs that are @@ -15,6 +15,8 @@ import logging import re from typing import Any, ClassVar +import httpx + logger = logging.getLogger(__name__) @@ -23,9 +25,20 @@ class GraphQLDiscovery: # Common paths where GraphQL endpoints live COMMON_PATHS: ClassVar[list[str]] = [ - "/graphql", "/api/graphql", "/api/v1/graphql", "/v1/graphql", "/v2/graphql", - "/graphql/v1", "/graphql/v2", "/gql", "/api/gql", "/query", "/api/query", - "/__graphql", "/altair", "/playground", + "/graphql", + "/api/graphql", + "/api/v1/graphql", + "/v1/graphql", + "/v2/graphql", + "/graphql/v1", + "/graphql/v2", + "/gql", + "/api/gql", + "/query", + "/api/query", + "/__graphql", + "/altair", + "/playground", ] # Patterns in JS bundles that indicate GraphQL endpoints @@ -65,9 +78,7 @@ class GraphQLDiscovery: for path in self.COMMON_PATHS: url = base_url.rstrip("/") + path try: - resp = await client.post( - url, json={"query": "{ __typename }"}, timeout=10 - ) + resp = await client.post(url, json={"query": "{ __typename }"}, timeout=10) if resp.is_success: try: data = resp.json() @@ -86,9 +97,7 @@ class GraphQLDiscovery: client = await get_client() try: - resp = await client.post( - endpoint, json={"query": self.INTROSPECTION_QUERY}, timeout=30 - ) + resp = await client.post(endpoint, json={"query": self.INTROSPECTION_QUERY}, timeout=30) if resp.is_success: return resp.json() except (httpx.HTTPError, httpx.RequestError) as e: diff --git a/intelligence.py b/intelligence.py index 993f03d..1771b12 100644 --- a/intelligence.py +++ b/intelligence.py @@ -1,22 +1,21 @@ """Pry — Competitive Intelligence Engine. Historical snapshots, anomaly detection, natural-language alerts, weekly reports.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging -import os import statistics import time from datetime import UTC, datetime from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) INTEL_DIR = PRY_DATA_DIR / "intel" diff --git a/llm_features.py b/llm_features.py index 4ac4bc8..8f5e3ff 100644 --- a/llm_features.py +++ b/llm_features.py @@ -20,9 +20,9 @@ def _strip_fence(text: str) -> str: """Strip markdown code fences that LLMs commonly wrap JSON in.""" t = text.strip() if t.startswith("```json"): - t = t[len("```json"):] + t = t[len("```json") :] elif t.startswith("```"): - t = t[len("```"):] + t = t[len("```") :] if t.endswith("```"): t = t[: -len("```")] return t.strip() diff --git a/llm_providers/base.py b/llm_providers/base.py index f84a032..bb496cb 100644 --- a/llm_providers/base.py +++ b/llm_providers/base.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) @dataclass class LLMResponse: """Standard response from any LLM provider.""" + text: str model: str provider: str @@ -33,6 +34,7 @@ class LLMResponse: @dataclass class ReferralConfig: """Referral/affiliate config for revenue sharing.""" + enabled: bool = True program_id: str = "pry-default" # Provider-specific referral links (with our affiliate ID) @@ -43,6 +45,7 @@ class ReferralConfig: def __post_init__(self): if not self.referral_links: from referrals import PROVIDER_CATALOG + for _category, providers in PROVIDER_CATALOG.items(): for p in providers: self.referral_links[p["tag"]] = p["url"] @@ -58,8 +61,14 @@ class LLMProvider(ABC): referral_url: str = "" @abstractmethod - async def complete(self, prompt: str, system: str = "", max_tokens: int = 1024, - temperature: float = 0.7, model: str = "") -> LLMResponse: + async def complete( + self, + prompt: str, + system: str = "", + max_tokens: int = 1024, + temperature: float = 0.7, + model: str = "", + ) -> LLMResponse: """Send completion request to provider.""" raise NotImplementedError @@ -69,4 +78,6 @@ class LLMProvider(ABC): raise NotImplementedError def estimate_cost(self, input_tokens: int, output_tokens: int) -> float: - return (input_tokens / 1000) * self.cost_per_1k_input + (output_tokens / 1000) * self.cost_per_1k_output + return (input_tokens / 1000) * self.cost_per_1k_input + ( + output_tokens / 1000 + ) * self.cost_per_1k_output diff --git a/llm_providers/providers.py b/llm_providers/providers.py index 43f7c68..7944250 100644 --- a/llm_providers/providers.py +++ b/llm_providers/providers.py @@ -22,27 +22,48 @@ class OpenAIProvider(LLMProvider): def __init__(self, api_key: str = ""): self.api_key = api_key or os.getenv("OPENAI_API_KEY", "") - async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model="gpt-4o-mini"): + async def complete( + self, prompt, system="", max_tokens=1024, temperature=0.7, model="gpt-4o-mini" + ): from client import get_client + client = await get_client() messages = [] - if system: messages.append({"role": "system", "content": system}) + if system: + messages.append({"role": "system", "content": system}) messages.append({"role": "user", "content": prompt}) - resp = await client.post("https://api.openai.com/v1/chat/completions", - json={"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature}, - headers={"Authorization": f"Bearer {self.api_key}"}, timeout=60) + resp = await client.post( + "https://api.openai.com/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + }, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=60, + ) data = resp.json() choice = data["choices"][0] - return LLMResponse(text=choice["message"]["content"], model=model, provider=self.name, - input_tokens=data["usage"]["prompt_tokens"], - output_tokens=data["usage"]["completion_tokens"], raw=data) + return LLMResponse( + text=choice["message"]["content"], + model=model, + provider=self.name, + input_tokens=data["usage"]["prompt_tokens"], + output_tokens=data["usage"]["completion_tokens"], + raw=data, + ) async def embed(self, text, model="text-embedding-3-small"): from client import get_client + client = await get_client() - resp = await client.post("https://api.openai.com/v1/embeddings", + resp = await client.post( + "https://api.openai.com/v1/embeddings", json={"input": text, "model": model}, - headers={"Authorization": f"Bearer {self.api_key}"}, timeout=30) + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=30, + ) return resp.json()["data"][0]["embedding"] @@ -55,18 +76,35 @@ class AnthropicProvider(LLMProvider): def __init__(self, api_key: str = ""): self.api_key = api_key or os.getenv("ANTHROPIC_API_KEY", "") - async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model="claude-3-haiku-20240307"): + async def complete( + self, prompt, system="", max_tokens=1024, temperature=0.7, model="claude-3-haiku-20240307" + ): from client import get_client + client = await get_client() - body = {"model": model, "max_tokens": max_tokens, "temperature": temperature, - "messages": [{"role": "user", "content": prompt}]} - if system: body["system"] = system - resp = await client.post("https://api.anthropic.com/v1/messages", - json=body, headers={"x-api-key": self.api_key, "anthropic-version": "2023-06-01"}, timeout=60) + body = { + "model": model, + "max_tokens": max_tokens, + "temperature": temperature, + "messages": [{"role": "user", "content": prompt}], + } + if system: + body["system"] = system + resp = await client.post( + "https://api.anthropic.com/v1/messages", + json=body, + headers={"x-api-key": self.api_key, "anthropic-version": "2023-06-01"}, + timeout=60, + ) data = resp.json() - return LLMResponse(text=data["content"][0]["text"], model=model, provider=self.name, - input_tokens=data["usage"]["input_tokens"], - output_tokens=data["usage"]["output_tokens"], raw=data) + return LLMResponse( + text=data["content"][0]["text"], + model=model, + provider=self.name, + input_tokens=data["usage"]["input_tokens"], + output_tokens=data["usage"]["output_tokens"], + raw=data, + ) async def embed(self, text, model=""): raise NotImplementedError("Anthropic doesn't have a public embedding API yet") @@ -81,23 +119,36 @@ class GoogleProvider(LLMProvider): def __init__(self, api_key: str = ""): self.api_key = api_key or os.getenv("GOOGLE_API_KEY", "") - async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model="gemini-1.5-flash"): + async def complete( + self, prompt, system="", max_tokens=1024, temperature=0.7, model="gemini-1.5-flash" + ): from client import get_client + client = await get_client() url = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent?key={self.api_key}" contents = [{"role": "user", "parts": [{"text": prompt}]}] - body = {"contents": contents, "generationConfig": {"maxOutputTokens": max_tokens, "temperature": temperature}} - if system: body["systemInstruction"] = {"parts": [{"text": system}]} + body = { + "contents": contents, + "generationConfig": {"maxOutputTokens": max_tokens, "temperature": temperature}, + } + if system: + body["systemInstruction"] = {"parts": [{"text": system}]} resp = await client.post(url, json=body, timeout=60) data = resp.json() text = data["candidates"][0]["content"]["parts"][0]["text"] usage = data.get("usageMetadata", {}) - return LLMResponse(text=text, model=model, provider=self.name, - input_tokens=usage.get("promptTokenCount", 0), - output_tokens=usage.get("candidatesTokenCount", 0), raw=data) + return LLMResponse( + text=text, + model=model, + provider=self.name, + input_tokens=usage.get("promptTokenCount", 0), + output_tokens=usage.get("candidatesTokenCount", 0), + raw=data, + ) async def embed(self, text, model="text-embedding-004"): from client import get_client + client = await get_client() url = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:embedContent?key={self.api_key}" resp = await client.post(url, json={"content": {"parts": [{"text": text}]}}, timeout=30) @@ -113,22 +164,39 @@ class CohereProvider(LLMProvider): def __init__(self, api_key: str = ""): self.api_key = api_key or os.getenv("COHERE_API_KEY", "") - async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model="command-r"): + async def complete( + self, prompt, system="", max_tokens=1024, temperature=0.7, model="command-r" + ): from client import get_client + client = await get_client() - body = {"model": model, "message": prompt, "max_tokens": max_tokens, "temperature": temperature} - if system: body["preamble"] = system - resp = await client.post("https://api.cohere.ai/v1/chat", - json=body, headers={"Authorization": f"Bearer {self.api_key}"}, timeout=60) + body = { + "model": model, + "message": prompt, + "max_tokens": max_tokens, + "temperature": temperature, + } + if system: + body["preamble"] = system + resp = await client.post( + "https://api.cohere.ai/v1/chat", + json=body, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=60, + ) data = resp.json() return LLMResponse(text=data["text"], model=model, provider=self.name, raw=data) async def embed(self, text, model="embed-english-v3.0"): from client import get_client + client = await get_client() - resp = await client.post("https://api.cohere.ai/v1/embed", + resp = await client.post( + "https://api.cohere.ai/v1/embed", json={"texts": [text], "model": model, "input_type": "search_document"}, - headers={"Authorization": f"Bearer {self.api_key}"}, timeout=30) + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=30, + ) return resp.json()["embeddings"][0] @@ -141,26 +209,47 @@ class MistralProvider(LLMProvider): def __init__(self, api_key: str = ""): self.api_key = api_key or os.getenv("MISTRAL_API_KEY", "") - async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model="mistral-small-latest"): + async def complete( + self, prompt, system="", max_tokens=1024, temperature=0.7, model="mistral-small-latest" + ): from client import get_client + client = await get_client() messages = [] - if system: messages.append({"role": "system", "content": system}) + if system: + messages.append({"role": "system", "content": system}) messages.append({"role": "user", "content": prompt}) - resp = await client.post("https://api.mistral.ai/v1/chat/completions", - json={"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature}, - headers={"Authorization": f"Bearer {self.api_key}"}, timeout=60) + resp = await client.post( + "https://api.mistral.ai/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + }, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=60, + ) data = resp.json() - return LLMResponse(text=data["choices"][0]["message"]["content"], model=model, provider=self.name, - input_tokens=data["usage"]["prompt_tokens"], - output_tokens=data["usage"]["completion_tokens"], raw=data) + return LLMResponse( + text=data["choices"][0]["message"]["content"], + model=model, + provider=self.name, + input_tokens=data["usage"]["prompt_tokens"], + output_tokens=data["usage"]["completion_tokens"], + raw=data, + ) async def embed(self, text, model="mistral-embed"): from client import get_client + client = await get_client() - resp = await client.post("https://api.mistral.ai/v1/embeddings", + resp = await client.post( + "https://api.mistral.ai/v1/embeddings", json={"model": model, "input": [text]}, - headers={"Authorization": f"Bearer {self.api_key}"}, timeout=30) + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=30, + ) return resp.json()["data"][0]["embedding"] @@ -176,22 +265,36 @@ class OllamaProvider(LLMProvider): async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model=""): from client import get_client + client = await get_client() model = model or self.default_model - body = {"model": model, "prompt": prompt, "stream": False, "options": {"temperature": temperature, "num_predict": max_tokens}} - if system: body["system"] = system + body = { + "model": model, + "prompt": prompt, + "stream": False, + "options": {"temperature": temperature, "num_predict": max_tokens}, + } + if system: + body["system"] = system resp = await client.post(f"{self.base_url}/api/generate", json=body, timeout=300) data = resp.json() - return LLMResponse(text=data["response"], model=model, provider=self.name, - input_tokens=data.get("prompt_eval_count", 0), - output_tokens=data.get("eval_count", 0), raw=data) + return LLMResponse( + text=data["response"], + model=model, + provider=self.name, + input_tokens=data.get("prompt_eval_count", 0), + output_tokens=data.get("eval_count", 0), + raw=data, + ) async def embed(self, text, model=""): from client import get_client + client = await get_client() model = model or "nomic-embed-text" - resp = await client.post(f"{self.base_url}/api/embeddings", - json={"model": model, "prompt": text}, timeout=60) + resp = await client.post( + f"{self.base_url}/api/embeddings", json={"model": model, "prompt": text}, timeout=60 + ) return resp.json()["embedding"] @@ -204,20 +307,42 @@ class OpenRouterProvider(LLMProvider): def __init__(self, api_key: str = ""): self.api_key = api_key or os.getenv("OPENROUTER_API_KEY", "") - async def complete(self, prompt, system="", max_tokens=1024, temperature=0.7, model="meta-llama/llama-3.2-3b-instruct:free"): + async def complete( + self, + prompt, + system="", + max_tokens=1024, + temperature=0.7, + model="meta-llama/llama-3.2-3b-instruct:free", + ): from client import get_client + client = await get_client() messages = [] - if system: messages.append({"role": "system", "content": system}) + if system: + messages.append({"role": "system", "content": system}) messages.append({"role": "user", "content": prompt}) - resp = await client.post("https://openrouter.ai/api/v1/chat/completions", - json={"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature}, - headers={"Authorization": f"Bearer {self.api_key}"}, timeout=60) + resp = await client.post( + "https://openrouter.ai/api/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + }, + headers={"Authorization": f"Bearer {self.api_key}"}, + timeout=60, + ) data = resp.json() usage = data.get("usage", {}) - return LLMResponse(text=data["choices"][0]["message"]["content"], model=model, provider=self.name, - input_tokens=usage.get("prompt_tokens", 0), - output_tokens=usage.get("completion_tokens", 0), raw=data) + return LLMResponse( + text=data["choices"][0]["message"]["content"], + model=model, + provider=self.name, + input_tokens=usage.get("prompt_tokens", 0), + output_tokens=usage.get("completion_tokens", 0), + raw=data, + ) async def embed(self, text, model=""): raise NotImplementedError("OpenRouter focuses on chat; use dedicated embedding providers") @@ -234,8 +359,12 @@ def register_default_providers(registry: "LLMRegistry") -> None: "openrouter": os.getenv("OPENROUTER_API_KEY"), } provider_classes = { - "openai": OpenAIProvider, "anthropic": AnthropicProvider, "google": GoogleProvider, - "cohere": CohereProvider, "mistral": MistralProvider, "openrouter": OpenRouterProvider, + "openai": OpenAIProvider, + "anthropic": AnthropicProvider, + "google": GoogleProvider, + "cohere": CohereProvider, + "mistral": MistralProvider, + "openrouter": OpenRouterProvider, } for name, cls in provider_classes.items(): key = api_key_map.get(name) diff --git a/llm_providers/registry.py b/llm_providers/registry.py index a0fab34..f4cee3d 100644 --- a/llm_providers/registry.py +++ b/llm_providers/registry.py @@ -7,16 +7,14 @@ import json import logging -import os import time from collections import defaultdict from datetime import UTC, datetime -from pathlib import Path from typing import Any from llm_providers.base import LLMProvider, LLMResponse, ReferralConfig - from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) USAGE_DIR = PRY_DATA_DIR / "llm_usage" @@ -31,10 +29,15 @@ class LLMRegistry: self.referral = referral or ReferralConfig() self.fallback_chain: list[str] = [] # Usage tracking - self.usage: dict[str, dict[str, Any]] = defaultdict(lambda: { - "calls": 0, "input_tokens": 0, "output_tokens": 0, "cost_usd": 0.0, - "last_used": None, - }) + self.usage: dict[str, dict[str, Any]] = defaultdict( + lambda: { + "calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "cost_usd": 0.0, + "last_used": None, + } + ) self._load_usage() def register(self, provider: LLMProvider) -> None: @@ -46,13 +49,26 @@ class LLMRegistry: def set_fallback_chain(self, chain: list[str]) -> None: self.fallback_chain = chain - async def complete(self, prompt: str, system: str = "", provider_name: str = "", - max_tokens: int = 1024, temperature: float = 0.7, model: str = "", - fallback: bool = True) -> LLMResponse: + async def complete( + self, + prompt: str, + system: str = "", + provider_name: str = "", + max_tokens: int = 1024, + temperature: float = 0.7, + model: str = "", + fallback: bool = True, + ) -> LLMResponse: """Complete via specified provider or fallback chain.""" names = [provider_name] if provider_name else list(self.fallback_chain) if not fallback: - names = [provider_name] if provider_name else [self.fallback_chain[0]] if self.fallback_chain else [] + names = ( + [provider_name] + if provider_name + else [self.fallback_chain[0]] + if self.fallback_chain + else [] + ) last_error = "" for name in names: @@ -63,14 +79,17 @@ class LLMRegistry: start = time.time() response = await provider.complete(prompt, system, max_tokens, temperature, model) response.latency_ms = int((time.time() - start) * 1000) - response.cost_usd = provider.estimate_cost(response.input_tokens, response.output_tokens) + response.cost_usd = provider.estimate_cost( + response.input_tokens, response.output_tokens + ) response.referral_id = self.referral.program_id self._track(response) return response except Exception as e: # noqa: BLE001 last_error = str(e) - logger.warning("llm_provider_failed", - extra={"provider": name, "error": str(e)[:100]}) + logger.warning( + "llm_provider_failed", extra={"provider": name, "error": str(e)[:100]} + ) if not fallback: break raise Exception(f"All LLM providers failed. Last: {last_error}") @@ -84,7 +103,9 @@ class LLMRegistry: try: return await provider.embed(text, model) except Exception as e: # noqa: BLE001 - logger.warning("embed_provider_failed", extra={"provider": name, "error": str(e)[:80]}) + logger.warning( + "embed_provider_failed", extra={"provider": name, "error": str(e)[:80]} + ) raise Exception("All embedding providers failed") def _track(self, response: LLMResponse) -> None: @@ -97,7 +118,8 @@ class LLMRegistry: # NEW: track in-app LLM usage as referral revenue try: from referrals import ReferralTracker - if not hasattr(self, '_referral_tracker'): + + if not hasattr(self, "_referral_tracker"): self._referral_tracker = ReferralTracker() # Estimate revenue at 5% of user cost (typical affiliate share) self._referral_tracker.track_in_app_usage(response.provider, response.cost_usd * 0.05) @@ -108,14 +130,20 @@ class LLMRegistry: path = USAGE_DIR / f"usage_{today}.jsonl" try: with open(path, "a") as f: - f.write(json.dumps({ - "ts": datetime.now(UTC).isoformat(), - "provider": response.provider, "model": response.model, - "input_tokens": response.input_tokens, - "output_tokens": response.output_tokens, - "cost_usd": response.cost_usd, - "referral_id": response.referral_id, - }) + "\n") + f.write( + json.dumps( + { + "ts": datetime.now(UTC).isoformat(), + "provider": response.provider, + "model": response.model, + "input_tokens": response.input_tokens, + "output_tokens": response.output_tokens, + "cost_usd": response.cost_usd, + "referral_id": response.referral_id, + } + ) + + "\n" + ) except OSError: pass @@ -157,5 +185,6 @@ def get_registry() -> LLMRegistry: _registry = LLMRegistry() # Register providers lazily (only if API keys are set) from llm_providers.providers import register_default_providers + register_default_providers(_registry) return _registry diff --git a/logging_config.py b/logging_config.py index 489e325..100a4cd 100644 --- a/logging_config.py +++ b/logging_config.py @@ -19,6 +19,7 @@ handler on the root logger (we do this in setup_logging()). Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper Licensed under MIT. See LICENSE. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -33,6 +34,7 @@ from typing import Any try: import structlog + _HAS_STRUCTLOG = True except ImportError: # pragma: no cover _HAS_STRUCTLOG = False @@ -42,7 +44,7 @@ except ImportError: # pragma: no cover DEFAULT_LEVEL = "INFO" SERVICE_NAME = "pry" LOG_FORMAT_ENV = "PRY_LOG_FORMAT" # "json" (default) or "console" -LOG_LEVEL_ENV = "PRY_LOG_LEVEL" # "DEBUG", "INFO", "WARNING", "ERROR" +LOG_LEVEL_ENV = "PRY_LOG_LEVEL" # "DEBUG", "INFO", "WARNING", "ERROR" _configured = False @@ -160,12 +162,33 @@ def _add_request_id_if_present(_, __, event_dict: dict[str, Any]) -> dict[str, A # the foreign_pre_chain so existing code keeps working. In strict mode # (PRY_LOG_STRICT_EXTRAS=1) the filtering is disabled and the stdlib # default behavior takes over. -_RESERVED_LOGRECORD_KEYS = frozenset({ - "name", "msg", "args", "levelname", "levelno", "pathname", "filename", - "module", "exc_info", "exc_text", "stack_info", "lineno", "funcName", - "created", "msecs", "relativeCreated", "thread", "threadName", - "processName", "process", "message", "asctime", "taskName", -}) +_RESERVED_LOGRECORD_KEYS = frozenset( + { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", + "asctime", + "taskName", + } +) def _filter_reserved_extras(_, __, event_dict: dict[str, Any]) -> dict[str, Any]: diff --git a/mconfig.py b/mconfig.py index 0934506..f39fbc2 100644 --- a/mconfig.py +++ b/mconfig.py @@ -131,5 +131,3 @@ class PryConfig: except OSError: pass return {"status": "ok", "config": self.to_dict()} - - diff --git a/mcp_production.py b/mcp_production.py index ff844f7..199bbe8 100644 --- a/mcp_production.py +++ b/mcp_production.py @@ -130,27 +130,34 @@ def set_active_mcp_server(server: Any) -> None: def notify_resource_changed(uri: str) -> None: """Notify all connected MCP clients that a resource has changed.""" - _send_mcp_notification({ - "jsonrpc": "2.0", - "method": "notifications/resources/updated", - "params": {"uri": uri}, - }) + _send_mcp_notification( + { + "jsonrpc": "2.0", + "method": "notifications/resources/updated", + "params": {"uri": uri}, + } + ) def notify_resource_list_changed() -> None: """Notify all connected MCP clients that the resource list changed.""" - _send_mcp_notification({ - "jsonrpc": "2.0", - "method": "notifications/resources/list_changed", - }) + _send_mcp_notification( + { + "jsonrpc": "2.0", + "method": "notifications/resources/list_changed", + } + ) def notify_tool_list_changed() -> None: """Notify all connected MCP clients that the tool list changed.""" - _send_mcp_notification({ - "jsonrpc": "2.0", - "method": "notifications/tools/list_changed", - }) + _send_mcp_notification( + { + "jsonrpc": "2.0", + "method": "notifications/tools/list_changed", + } + ) + # Try to import official MCP SDK try: @@ -510,8 +517,16 @@ PRY_PROMPTS = [ "title": "Research a Company", "description": "Scrape and analyze a company website for competitive intelligence. Use the pry_scrape and pry_enrich tools to gather data, then summarize findings.", "arguments": [ - {"name": "company_name", "description": "Company name (e.g., 'Anthropic')", "required": True}, - {"name": "website", "description": "Company website URL (e.g., 'https://anthropic.com')", "required": True}, + { + "name": "company_name", + "description": "Company name (e.g., 'Anthropic')", + "required": True, + }, + { + "name": "website", + "description": "Company website URL (e.g., 'https://anthropic.com')", + "required": True, + }, ], }, { @@ -519,8 +534,16 @@ PRY_PROMPTS = [ "title": "Compare Products", "description": "Scrape product pages from multiple e-commerce sites and create a side-by-side comparison. Use pry_template tool with amazon-product, walmart-product, etc.", "arguments": [ - {"name": "product_query", "description": "What to search for (e.g., 'wireless headphones')", "required": True}, - {"name": "sites", "description": "Sites to compare (e.g., ['amazon', 'walmart', 'target'])", "required": True}, + { + "name": "product_query", + "description": "What to search for (e.g., 'wireless headphones')", + "required": True, + }, + { + "name": "sites", + "description": "Sites to compare (e.g., ['amazon', 'walmart', 'target'])", + "required": True, + }, ], }, { @@ -529,7 +552,11 @@ PRY_PROMPTS = [ "description": "Crawl a website and extract all email addresses, phone numbers, and social media handles using CSS selectors and regex patterns.", "arguments": [ {"name": "url", "description": "Starting URL", "required": True}, - {"name": "max_pages", "description": "Maximum pages to crawl (default: 20)", "required": False}, + { + "name": "max_pages", + "description": "Maximum pages to crawl (default: 20)", + "required": False, + }, ], }, { @@ -538,7 +565,11 @@ PRY_PROMPTS = [ "description": "Set up continuous monitoring for a competitor's pricing, product listings, or content. Get notified via webhook on changes.", "arguments": [ {"name": "url", "description": "Competitor URL to monitor", "required": True}, - {"name": "what_to_track", "description": "What to track (e.g., 'product prices', 'blog posts', 'job listings')", "required": True}, + { + "name": "what_to_track", + "description": "What to track (e.g., 'product prices', 'blog posts', 'job listings')", + "required": True, + }, {"name": "webhook_url", "description": "Where to send notifications", "required": True}, ], }, @@ -548,7 +579,11 @@ PRY_PROMPTS = [ "description": "Scrape an article, summarize key points, and provide structured analysis (sentiment, entities, key claims).", "arguments": [ {"name": "url", "description": "Article URL", "required": True}, - {"name": "focus", "description": "What to focus on (e.g., 'financial implications', 'competitive threats')", "required": False}, + { + "name": "focus", + "description": "What to focus on (e.g., 'financial implications', 'competitive threats')", + "required": False, + }, ], }, { @@ -557,7 +592,11 @@ PRY_PROMPTS = [ "description": "Find and extract a specific data table from a webpage. Returns structured CSV/JSON rows.", "arguments": [ {"name": "url", "description": "Page URL", "required": True}, - {"name": "table_description", "description": "Description of the table you want (e.g., 'product pricing table', 'league standings')", "required": True}, + { + "name": "table_description", + "description": "Description of the table you want (e.g., 'product pricing table', 'league standings')", + "required": True, + }, ], }, { @@ -566,7 +605,11 @@ PRY_PROMPTS = [ "description": "Scrape any URL with a custom JSON schema. Use pry_extract or pry_scrape with a schema definition.", "arguments": [ {"name": "url", "description": "URL to scrape", "required": True}, - {"name": "fields", "description": "Comma-separated list of fields to extract (e.g., 'title,price,description,image')", "required": True}, + { + "name": "fields", + "description": "Comma-separated list of fields to extract (e.g., 'title,price,description,image')", + "required": True, + }, ], }, ] @@ -574,7 +617,8 @@ PRY_PROMPTS = [ def make_fallback_server( base_url: str = DEFAULT_BASE_URL, - tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]] | None = None, + tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]] + | None = None, ) -> Any: """Build a minimal stdio JSON-RPC server if official MCP SDK not available. This implements the MCP 2024-11-05 protocol with all required methods.""" @@ -583,7 +627,8 @@ def make_fallback_server( def __init__( self, base_url: str = DEFAULT_BASE_URL, - tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]] | None = None, + tool_executor: Callable[[str, dict[str, Any]], Coroutine[Any, Any, dict[str, Any]]] + | None = None, ) -> None: self.tools: dict[str, dict[str, Any]] = {} self.resources: dict[str, dict[str, Any]] = {} @@ -656,7 +701,9 @@ def make_fallback_server( result = await self.tool_executor(name, arguments) return self._tool_result_to_content(result, name) except Exception as e: # noqa: BLE001 - logger.warning("mcp_tool_executor_failed", extra={"tool": name, "error": str(e)}) + logger.warning( + "mcp_tool_executor_failed", extra={"tool": name, "error": str(e)} + ) return { "content": [{"type": "text", "text": f"Tool {name} failed: {e}"}], "isError": True, @@ -744,36 +791,45 @@ def make_fallback_server( if uri == "pry://catalog": try: from template_engine import list_templates + templates = list_templates() categories: dict[str, list[str]] = {} for t in templates: cat = t.get("category", "general") categories.setdefault(cat, []).append(t.get("template_id", "unknown")) - return json.dumps({ - "total_templates": len(templates), - "categories": categories, - "note": "Use pry_search_templates to query dynamically.", - }, indent=2) + return json.dumps( + { + "total_templates": len(templates), + "categories": categories, + "note": "Use pry_search_templates to query dynamically.", + }, + indent=2, + ) except (json.JSONDecodeError, ValueError) as e: return json.dumps({"error": f"Could not load template catalog: {e}"}) if uri == "pry://stats": - return json.dumps({ - "server": SERVER_NAME, - "version": SERVER_VERSION, - "protocol_version": MCP_PROTOCOL_VERSION, - "tools_count": len(self.tools), - "resources_count": len(self.resources), - "prompts_count": len(self.prompts), - }, indent=2) + return json.dumps( + { + "server": SERVER_NAME, + "version": SERVER_VERSION, + "protocol_version": MCP_PROTOCOL_VERSION, + "tools_count": len(self.tools), + "resources_count": len(self.resources), + "prompts_count": len(self.prompts), + }, + indent=2, + ) if uri == "pry://x402/pricing": try: from x402 import X402Handler + return json.dumps(X402Handler().get_stats(), indent=2) except (json.JSONDecodeError, ValueError) as e: return json.dumps({"error": f"Could not load x402 pricing: {e}"}) if uri == "pry://referrals": try: from referrals import ReferralTracker + return json.dumps(ReferralTracker().get_catalog(), indent=2) # type: ignore[no-untyped-call] except (json.JSONDecodeError, ValueError) as e: return json.dumps({"error": f"Could not load referrals: {e}"}) @@ -795,110 +851,126 @@ def make_fallback_server( if name == "research_company": company = arguments.get("company_name", "the company") website = arguments.get("website", "") - return [{ - "role": "user", - "content": { - "type": "text", - "text": ( - f"Research {company}. " - f"{'Start by scraping ' + website + '. ' if website else ''}" - "Use pry_scrape and pry_enrich to gather data, then summarize: " - "what they do, target market, key products, competitive positioning, " - "and any risks or opportunities." - ), - }, - }] + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Research {company}. " + f"{'Start by scraping ' + website + '. ' if website else ''}" + "Use pry_scrape and pry_enrich to gather data, then summarize: " + "what they do, target market, key products, competitive positioning, " + "and any risks or opportunities." + ), + }, + } + ] if name == "compare_products": query = arguments.get("product_query", "the product") sites = arguments.get("sites", []) - return [{ - "role": "user", - "content": { - "type": "text", - "text": ( - f"Compare prices and details for '{query}' across {', '.join(sites)}. " - "Use pry_template with amazon-product, walmart-product, etc. " - "Return a side-by-side comparison table." - ), - }, - }] + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Compare prices and details for '{query}' across {', '.join(sites)}. " + "Use pry_template with amazon-product, walmart-product, etc. " + "Return a side-by-side comparison table." + ), + }, + } + ] if name == "extract_contacts": url = arguments.get("url", "") max_pages = arguments.get("max_pages", 20) - return [{ - "role": "user", - "content": { - "type": "text", - "text": ( - f"Extract all contact information from {url}. " - f"Crawl up to {max_pages} pages. " - "Use pry_crawl and regex to find emails, phones, and social profiles." - ), - }, - }] + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Extract all contact information from {url}. " + f"Crawl up to {max_pages} pages. " + "Use pry_crawl and regex to find emails, phones, and social profiles." + ), + }, + } + ] if name == "monitor_competitor": url = arguments.get("url", "") what = arguments.get("what_to_track", "changes") webhook = arguments.get("webhook_url", "") - return [{ - "role": "user", - "content": { - "type": "text", - "text": ( - f"Set up monitoring for {url}. Track {what}. " - f"{'Notify via webhook: ' + webhook + '. ' if webhook else ''}" - "Use pry_monitor." - ), - }, - }] + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Set up monitoring for {url}. Track {what}. " + f"{'Notify via webhook: ' + webhook + '. ' if webhook else ''}" + "Use pry_monitor." + ), + }, + } + ] if name == "analyze_article": url = arguments.get("url", "") focus = arguments.get("focus", "") - return [{ - "role": "user", - "content": { - "type": "text", - "text": ( - f"Scrape and analyze this article: {url}. " - f"{'Focus on: ' + focus + '. ' if focus else ''}" - "Summarize key points, sentiment, entities, and claims." - ), - }, - }] + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Scrape and analyze this article: {url}. " + f"{'Focus on: ' + focus + '. ' if focus else ''}" + "Summarize key points, sentiment, entities, and claims." + ), + }, + } + ] if name == "extract_table": url = arguments.get("url", "") desc = arguments.get("table_description", "") - return [{ - "role": "user", - "content": { - "type": "text", - "text": ( - f"Extract the data table from {url}. Table: {desc}. " - "Use pry_extract with CSS selectors or pry_scrape + structured output." - ), - }, - }] + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Extract the data table from {url}. Table: {desc}. " + "Use pry_extract with CSS selectors or pry_scrape + structured output." + ), + }, + } + ] if name == "scrape_with_schema": url = arguments.get("url", "") fields = arguments.get("fields", "") - return [{ + return [ + { + "role": "user", + "content": { + "type": "text", + "text": ( + f"Scrape {url} and extract these fields: {fields}. " + "Use pry_extract with a custom JSON schema." + ), + }, + } + ] + arg_text = ", ".join(f"{k}={v}" for k, v in arguments.items()) + return [ + { "role": "user", "content": { "type": "text", - "text": ( - f"Scrape {url} and extract these fields: {fields}. " - "Use pry_extract with a custom JSON schema." - ), + "text": f"Execute the {self.prompts[name].get('title', name)} prompt with arguments: {arg_text}", }, - }] - arg_text = ", ".join(f"{k}={v}" for k, v in arguments.items()) - return [{ - "role": "user", - "content": { - "type": "text", - "text": f"Execute the {self.prompts[name].get('title', name)} prompt with arguments: {arg_text}", - }, - }] + } + ] def complete(self, ref: dict[str, Any], argument: dict[str, Any]) -> dict[str, Any]: """Provide argument completion suggestions.""" @@ -907,11 +979,16 @@ def make_fallback_server( arg_name = argument.get("name", "") arg_value = argument.get("value", "") values: list[str] = [] - if ref_type == "ref/prompt" and ref_name == "extract_contacts" and arg_name == "max_pages": + if ( + ref_type == "ref/prompt" + and ref_name == "extract_contacts" + and arg_name == "max_pages" + ): values = ["10", "20", "50", "100"] elif ref_type == "ref/tool" and arg_name == "template_id": try: from template_engine import list_templates + templates = list_templates() values = [ t["template_id"] @@ -963,7 +1040,9 @@ def make_fallback_server( "result": {"tools": list(self.tools.values())}, } if method == "tools/call": - tool_result = await self.call_tool(params.get("name", ""), params.get("arguments", {})) + tool_result = await self.call_tool( + params.get("name", ""), params.get("arguments", {}) + ) if "error" in tool_result: return { "jsonrpc": "2.0", diff --git a/mcp_server.py b/mcp_server.py index ccbadad..544c276 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -129,5 +129,3 @@ class PryMCPServer: async with httpx.AsyncClient(timeout=120) as client: resp = await client.post(f"{self.base_url}{path}", json=payload) return resp.json() - - diff --git a/monitor.py b/monitor.py index 66ff60c..ccac670 100644 --- a/monitor.py +++ b/monitor.py @@ -1,14 +1,11 @@ -import httpx + """Pry — scheduled monitors with AI change detection. Cron-based monitors that diff content and judge meaningful changes.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import difflib import json import logging @@ -20,6 +17,10 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +import httpx + +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) MONITORS_DIR = PRY_DATA_DIR / "monitors" diff --git a/observability.py b/observability.py index fc131d3..ca0c82a 100644 --- a/observability.py +++ b/observability.py @@ -14,6 +14,7 @@ logger = logging.getLogger(__name__) # Try to import Prometheus try: from prometheus_client import CONTENT_TYPE_LATEST, Counter, Gauge, Histogram, generate_latest + _has_prometheus = True except ImportError: _has_prometheus = False @@ -23,6 +24,7 @@ try: from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + _has_otel = True except ImportError: _has_otel = False @@ -30,7 +32,9 @@ except ImportError: # Metrics if _has_prometheus: - REQUEST_COUNT = Counter("pry_requests_total", "Total requests", ["method", "endpoint", "status"]) + REQUEST_COUNT = Counter( + "pry_requests_total", "Total requests", ["method", "endpoint", "status"] + ) REQUEST_LATENCY = Histogram("pry_request_duration_seconds", "Request latency", ["endpoint"]) SCRAPE_COUNT = Counter("pry_scrapes_total", "Total scrapes", ["method", "status"]) SCRAPE_LATENCY = Histogram("pry_scrape_duration_seconds", "Scrape latency", ["method"]) @@ -46,7 +50,8 @@ else: def setup_tracing(service_name: str = "pry") -> None: """Initialize OpenTelemetry tracing.""" - if not _has_otel: return + if not _has_otel: + return try: provider = TracerProvider() processor = SimpleSpanProcessor(ConsoleSpanExporter()) @@ -64,7 +69,7 @@ def track_request(endpoint: str, method: str = "GET"): status = "success" try: yield - except Exception: # noqa: BLE001 + except Exception: status = "error" raise finally: @@ -81,7 +86,7 @@ def track_scrape(method: str = "direct"): status = "success" try: yield - except Exception: # noqa: BLE001 + except Exception: status = "error" raise finally: diff --git a/ocr_extractor.py b/ocr_extractor.py index 658f182..3bfe68f 100644 --- a/ocr_extractor.py +++ b/ocr_extractor.py @@ -1,4 +1,4 @@ -import httpx + """Pry — Image OCR using Tesseract. Extract text from images on web pages. Uses pytesseract + Pillow.""" @@ -13,6 +13,8 @@ import logging import os from typing import Any, ClassVar +import httpx + logger = logging.getLogger(__name__) try: @@ -28,8 +30,18 @@ class ImageOCR: """Extract text from images using Tesseract OCR.""" SUPPORTED_LANGUAGES: ClassVar[list[str]] = [ - "eng", "chi_sim", "chi_tra", "spa", "fra", "deu", "ita", - "por", "rus", "jpn", "kor", "ara", + "eng", + "chi_sim", + "chi_tra", + "spa", + "fra", + "deu", + "ita", + "por", + "rus", + "jpn", + "kor", + "ara", ] def __init__(self, language: str = "eng"): diff --git a/parser.py b/parser.py index e594e60..85a4eec 100644 --- a/parser.py +++ b/parser.py @@ -10,6 +10,7 @@ Temp files are always cleaned up in finally blocks. # Licensed under MIT. See LICENSE. import asyncio +import contextlib import io import os import tempfile @@ -121,7 +122,5 @@ class DocumentParser: return {"text": f"[Image OCR failed: {e}]", "format": "image", "pages": 0} finally: if fname and os.path.exists(fname): - try: + with contextlib.suppress(OSError): os.unlink(fname) - except OSError: - pass diff --git a/paths.py b/paths.py index c910685..0b27025 100644 --- a/paths.py +++ b/paths.py @@ -13,6 +13,7 @@ of hardcoding Path("~/.pry/x"). Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper Licensed under MIT. See LICENSE. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # diff --git a/pipeline.py b/pipeline.py index 054cbc3..10f83f7 100644 --- a/pipeline.py +++ b/pipeline.py @@ -76,7 +76,7 @@ class Pipeline: result = cast(SyncHookFn, fn)(**context) if isinstance(result, dict): context.update(result) - except Exception as e: # noqa: BLE001 + except Exception as e: logger.exception( "hook_failed", extra={"point": hook_point, "fn": getattr(fn, "__name__", str(fn))}, diff --git a/pipelines.py b/pipelines.py index 0b79bf7..72b7123 100644 --- a/pipelines.py +++ b/pipelines.py @@ -3,22 +3,21 @@ JSON-defined workflow engine. Users define pipelines as structured steps, the engine executes them sequentially with branching and error handling. A UI can render these steps as drag-and-drop blocks.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging import os import uuid from datetime import UTC, datetime -from pathlib import Path from typing import Any, cast +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) PIPELINE_DIR = PRY_DATA_DIR / "pipelines" @@ -459,7 +458,8 @@ def save_pipeline(pipeline: dict[str, Any]) -> dict[str, Any]: try: path.write_text(json.dumps(pipeline, indent=2)) logger.info( - "pipeline_saved", extra={"pipeline_id": pipeline_id, "pipeline_name": pipeline.get("name")} + "pipeline_saved", + extra={"pipeline_id": pipeline_id, "pipeline_name": pipeline.get("name")}, ) return {"success": True, "pipeline_id": pipeline_id} except OSError as e: diff --git a/proxy_manager.py b/proxy_manager.py index 6b90c76..8728630 100644 --- a/proxy_manager.py +++ b/proxy_manager.py @@ -1,14 +1,12 @@ """Pry — Proxy Manager with affiliate-signup flow. Tries free proxies first (Tor, public rotating). When premium proxy is needed, prompts user to sign up via affiliate links. Pre-wired to 5+ providers.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging import os @@ -18,6 +16,8 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) try: @@ -465,7 +465,6 @@ class ProxyManager: logger.debug("recent_clicks_failed", extra={"error": str(e)[:80]}) return [] - # ── Proxy referral catalog (from proxy_referrals.py) ──────────────── def get_proxy_referral(self, provider_tag: str) -> dict[str, Any]: """Get the curated referral info for a proxy provider. @@ -487,9 +486,7 @@ class ProxyManager: def list_proxy_referrals(self) -> list[dict[str, Any]]: """List all proxy provider referrals (curated, marketing-friendly view).""" - return [ - {"tag": tag, **info} for tag, info in _PROXY_REFERRALS.items() - ] + return [{"tag": tag, **info} for tag, info in _PROXY_REFERRALS.items()] def get_proxy_referral_summary(self) -> dict[str, Any]: """Return a per-tier summary of proxy referrals (count, providers, total commission ceiling).""" @@ -504,7 +501,6 @@ class ProxyManager: } - def _parse_ts(ts: str) -> float: """Parse an ISO timestamp to epoch seconds. Returns 0 on error.""" if not ts: diff --git a/proxy_referrals.py b/proxy_referrals.py index 04f3bf7..1482e24 100644 --- a/proxy_referrals.py +++ b/proxy_referrals.py @@ -5,6 +5,7 @@ # Licensed under MIT. See LICENSE. """Proxy provider referral links — monetize Pry by referring users to proxy services.""" + from __future__ import annotations # Each provider has a referral URL and commission structure diff --git a/quality.py b/quality.py index 06bec34..40f881a 100644 --- a/quality.py +++ b/quality.py @@ -1,24 +1,22 @@ """Pry — Data Quality SLA Dashboard. Per-extraction quality metrics, anomaly detection, freshness tracking.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import difflib import hashlib import json import logging -import os import time from contextlib import suppress from datetime import UTC, datetime -from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) QUALITY_DIR = PRY_DATA_DIR / "quality" diff --git a/reconciliation.py b/reconciliation.py index 1a0c8a1..adb2a40 100644 --- a/reconciliation.py +++ b/reconciliation.py @@ -362,9 +362,9 @@ def build_reconciliation_report( # ── API helpers ── - - -async def llm_enhance_reconciliation(entities: list[dict[str, Any]], low_confidence_threshold: float = 0.5) -> dict[str, Any]: +async def llm_enhance_reconciliation( + entities: list[dict[str, Any]], low_confidence_threshold: float = 0.5 +) -> dict[str, Any]: """Use the LLM to verify or refute low-confidence entity matches. For each entity group whose field-based confidence is below the threshold, @@ -377,6 +377,7 @@ async def llm_enhance_reconciliation(entities: list[dict[str, Any]], low_confide """ try: from llm_features import llm_entity_reconcile + low_conf = [e for e in entities if e.get("confidence", 1.0) < low_confidence_threshold] if not low_conf: return {"llm_enhanced": False, "verified": 0, "refuted": 0, "low_confidence_groups": 0} @@ -403,6 +404,7 @@ async def llm_enhance_reconciliation(entities: list[dict[str, Any]], low_confide logger.debug("llm_reconciliation_failed", extra={"error": str(e)[:80]}) return {"llm_enhanced": False, "error": str(e)[:200]} + async def reconcile( records: list[dict[str, Any]], vertical: str, diff --git a/referrals.py b/referrals.py index fce2e37..4aa8724 100644 --- a/referrals.py +++ b/referrals.py @@ -1,22 +1,20 @@ """Pry — Referral / Affiliate revenue system. Tracks clicks, conversions, and revenue across 60+ providers. Supports x402 (HTTP 402) pay-per-scrape protocol for monetization.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging -import os import uuid from datetime import UTC, datetime, timedelta -from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) REFERRAL_DIR = PRY_DATA_DIR / "referrals" @@ -26,194 +24,563 @@ REFERRAL_DIR.mkdir(parents=True, exist_ok=True) # Format: { category: [ {name, url, commission, cookie_days, ...} ] } PROVIDER_CATALOG: dict[str, list[dict[str, Any]]] = { "llm": [ - {"name": "OpenAI", "url": "https://platform.openai.com/signup?via=pry", - "commission": "20% first 6 months", "cookie_days": 30, "tag": "openai"}, - {"name": "Anthropic", "url": "https://console.anthropic.com/?ref=pry", - "commission": "Enterprise referrals", "cookie_days": 30, "tag": "anthropic"}, - {"name": "Google AI Studio", "url": "https://aistudio.google.com/?utm_source=pry", - "commission": "Google Cloud Partner tiered", "cookie_days": 30, "tag": "google"}, - {"name": "Cohere", "url": "https://dashboard.cohere.com/welcome?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "cohere"}, - {"name": "Mistral AI", "url": "https://console.mistral.ai/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "mistral"}, - {"name": "OpenRouter", "url": "https://openrouter.ai/?ref=pry", - "commission": "$1/signup + usage share", "cookie_days": 30, "tag": "openrouter"}, - {"name": "Groq", "url": "https://console.groq.com/?ref=pry", - "commission": "Developer program", "cookie_days": 30, "tag": "groq"}, - {"name": "Together AI", "url": "https://api.together.xyz/?ref=pry", - "commission": "$5/signup + usage share", "cookie_days": 30, "tag": "together"}, - {"name": "Replicate", "url": "https://replicate.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "replicate"}, - {"name": "Fireworks AI", "url": "https://fireworks.ai/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "fireworks"}, - {"name": "xAI (Grok)", "url": "https://console.x.ai/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "xai"}, - {"name": "DeepInfra", "url": "https://deepinfra.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "deepinfra"}, - {"name": "Novita AI", "url": "https://novita.ai/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "novita"}, - {"name": "Anyscale", "url": "https://anyscale.com/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "anyscale"}, - {"name": "Lepton AI", "url": "https://www.lepton.ai/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "lepton"}, + { + "name": "OpenAI", + "url": "https://platform.openai.com/signup?via=pry", + "commission": "20% first 6 months", + "cookie_days": 30, + "tag": "openai", + }, + { + "name": "Anthropic", + "url": "https://console.anthropic.com/?ref=pry", + "commission": "Enterprise referrals", + "cookie_days": 30, + "tag": "anthropic", + }, + { + "name": "Google AI Studio", + "url": "https://aistudio.google.com/?utm_source=pry", + "commission": "Google Cloud Partner tiered", + "cookie_days": 30, + "tag": "google", + }, + { + "name": "Cohere", + "url": "https://dashboard.cohere.com/welcome?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "cohere", + }, + { + "name": "Mistral AI", + "url": "https://console.mistral.ai/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "mistral", + }, + { + "name": "OpenRouter", + "url": "https://openrouter.ai/?ref=pry", + "commission": "$1/signup + usage share", + "cookie_days": 30, + "tag": "openrouter", + }, + { + "name": "Groq", + "url": "https://console.groq.com/?ref=pry", + "commission": "Developer program", + "cookie_days": 30, + "tag": "groq", + }, + { + "name": "Together AI", + "url": "https://api.together.xyz/?ref=pry", + "commission": "$5/signup + usage share", + "cookie_days": 30, + "tag": "together", + }, + { + "name": "Replicate", + "url": "https://replicate.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "replicate", + }, + { + "name": "Fireworks AI", + "url": "https://fireworks.ai/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "fireworks", + }, + { + "name": "xAI (Grok)", + "url": "https://console.x.ai/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "xai", + }, + { + "name": "DeepInfra", + "url": "https://deepinfra.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "deepinfra", + }, + { + "name": "Novita AI", + "url": "https://novita.ai/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "novita", + }, + { + "name": "Anyscale", + "url": "https://anyscale.com/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "anyscale", + }, + { + "name": "Lepton AI", + "url": "https://www.lepton.ai/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "lepton", + }, ], "hosting": [ - {"name": "Hetzner", "url": "https://hetzner.com/?ref=pry", - "commission": "€25/signup + 10% recurring", "cookie_days": 90, "tag": "hetzner"}, - {"name": "DigitalOcean", "url": "https://m.do.co/c/pry", - "commission": "$25/paid referral", "cookie_days": 60, "tag": "do"}, - {"name": "Linode (Akamai)", "url": "https://www.linode.com/?ref=pry", - "commission": "$20/paid referral", "cookie_days": 60, "tag": "linode"}, - {"name": "Vultr", "url": "https://www.vultr.com/?ref=pry", - "commission": "$10-$50/signup", "cookie_days": 30, "tag": "vultr"}, - {"name": "OVHcloud", "url": "https://www.ovhcloud.com/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "ovh"}, - {"name": "UpCloud", "url": "https://upcloud.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "upcloud"}, - {"name": "Netcup", "url": "https://netcup.com/?ref=pry", - "commission": "€5/signup", "cookie_days": 90, "tag": "netcup"}, + { + "name": "Hetzner", + "url": "https://hetzner.com/?ref=pry", + "commission": "€25/signup + 10% recurring", + "cookie_days": 90, + "tag": "hetzner", + }, + { + "name": "DigitalOcean", + "url": "https://m.do.co/c/pry", + "commission": "$25/paid referral", + "cookie_days": 60, + "tag": "do", + }, + { + "name": "Linode (Akamai)", + "url": "https://www.linode.com/?ref=pry", + "commission": "$20/paid referral", + "cookie_days": 60, + "tag": "linode", + }, + { + "name": "Vultr", + "url": "https://www.vultr.com/?ref=pry", + "commission": "$10-$50/signup", + "cookie_days": 30, + "tag": "vultr", + }, + { + "name": "OVHcloud", + "url": "https://www.ovhcloud.com/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "ovh", + }, + { + "name": "UpCloud", + "url": "https://upcloud.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "upcloud", + }, + { + "name": "Netcup", + "url": "https://netcup.com/?ref=pry", + "commission": "€5/signup", + "cookie_days": 90, + "tag": "netcup", + }, ], "domains": [ - {"name": "Namecheap", "url": "https://namecheap.com/?affiliateid=pry", - "commission": "Up to $50/domain", "cookie_days": 365, "tag": "namecheap"}, - {"name": "Cloudflare Registrar", "url": "https://cloudflare.com/?ref=pry", - "commission": "At-cost pricing", "cookie_days": 0, "tag": "cloudflare"}, - {"name": "Porkbun", "url": "https://porkbun.com/?affiliate=pry", - "commission": "$0.50-$2/domain", "cookie_days": 365, "tag": "porkbun"}, - {"name": "Spaceship", "url": "https://spaceship.com/?affiliate=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "spaceship"}, + { + "name": "Namecheap", + "url": "https://namecheap.com/?affiliateid=pry", + "commission": "Up to $50/domain", + "cookie_days": 365, + "tag": "namecheap", + }, + { + "name": "Cloudflare Registrar", + "url": "https://cloudflare.com/?ref=pry", + "commission": "At-cost pricing", + "cookie_days": 0, + "tag": "cloudflare", + }, + { + "name": "Porkbun", + "url": "https://porkbun.com/?affiliate=pry", + "commission": "$0.50-$2/domain", + "cookie_days": 365, + "tag": "porkbun", + }, + { + "name": "Spaceship", + "url": "https://spaceship.com/?affiliate=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "spaceship", + }, ], "cdn": [ - {"name": "Cloudflare", "url": "https://cloudflare.com/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "cloudflare"}, - {"name": "Bunny CDN", "url": "https://bunny.net?affiliate=pry", - "commission": "10% recurring lifetime", "cookie_days": 60, "tag": "bunny"}, - {"name": "Fastly", "url": "https://fastly.com/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "fastly"}, + { + "name": "Cloudflare", + "url": "https://cloudflare.com/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "cloudflare", + }, + { + "name": "Bunny CDN", + "url": "https://bunny.net?affiliate=pry", + "commission": "10% recurring lifetime", + "cookie_days": 60, + "tag": "bunny", + }, + { + "name": "Fastly", + "url": "https://fastly.com/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "fastly", + }, ], "email": [ - {"name": "SendGrid", "url": "https://sendgrid.com/?ref=pry", - "commission": "$30/referral", "cookie_days": 30, "tag": "sendgrid"}, - {"name": "Mailgun", "url": "https://mailgun.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "mailgun"}, - {"name": "Resend", "url": "https://resend.com/?ref=pry", - "commission": "$20/signup", "cookie_days": 30, "tag": "resend"}, - {"name": "Postmark", "url": "https://postmarkapp.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "postmark"}, - {"name": "Brevo (Sendinblue)", "url": "https://www.brevo.com/?ref=pry", - "commission": "Paid plan share", "cookie_days": 30, "tag": "brevo"}, + { + "name": "SendGrid", + "url": "https://sendgrid.com/?ref=pry", + "commission": "$30/referral", + "cookie_days": 30, + "tag": "sendgrid", + }, + { + "name": "Mailgun", + "url": "https://mailgun.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "mailgun", + }, + { + "name": "Resend", + "url": "https://resend.com/?ref=pry", + "commission": "$20/signup", + "cookie_days": 30, + "tag": "resend", + }, + { + "name": "Postmark", + "url": "https://postmarkapp.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "postmark", + }, + { + "name": "Brevo (Sendinblue)", + "url": "https://www.brevo.com/?ref=pry", + "commission": "Paid plan share", + "cookie_days": 30, + "tag": "brevo", + }, ], "monitoring": [ - {"name": "Sentry", "url": "https://sentry.io/?ref=pry", - "commission": "$100-$1,000 (tiered)", "cookie_days": 60, "tag": "sentry"}, - {"name": "Datadog", "url": "https://datadoghq.com/?ref=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "datadog"}, - {"name": "Better Stack", "url": "https://betterstack.com/?ref=pry", - "commission": "$50/paid signup", "cookie_days": 30, "tag": "betterstack"}, - {"name": "Highlight.io", "url": "https://highlight.io/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "highlight"}, - {"name": "PostHog", "url": "https://posthog.com/?ref=pry", - "commission": "20% recurring for 2 years", "cookie_days": 60, "tag": "posthog"}, + { + "name": "Sentry", + "url": "https://sentry.io/?ref=pry", + "commission": "$100-$1,000 (tiered)", + "cookie_days": 60, + "tag": "sentry", + }, + { + "name": "Datadog", + "url": "https://datadoghq.com/?ref=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "datadog", + }, + { + "name": "Better Stack", + "url": "https://betterstack.com/?ref=pry", + "commission": "$50/paid signup", + "cookie_days": 30, + "tag": "betterstack", + }, + { + "name": "Highlight.io", + "url": "https://highlight.io/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "highlight", + }, + { + "name": "PostHog", + "url": "https://posthog.com/?ref=pry", + "commission": "20% recurring for 2 years", + "cookie_days": 60, + "tag": "posthog", + }, ], "proxies": [ - {"name": "Bright Data", "url": "https://brightdata.com/cp/start?aff_id=pry", - "commission": "$2-$50/signup, up to 10% recurring", "cookie_days": 30, "tag": "brightdata", - "free_trial": "$5 credit", "min_price": "$0.10/GB residential", - "note": "Industry leader. Best for high-volume scraping."}, - {"name": "Smartproxy", "url": "https://dashboard.smartproxy.com/registration?aff_id=pry", - "commission": "20% recurring lifetime", "cookie_days": 60, "tag": "smartproxy", - "free_trial": "100MB free", "min_price": "$0.10/GB residential", - "note": "Good balance of price and quality."}, - {"name": "Oxylabs", "url": "https://dashboard.oxylabs.io/?aff_id=pry", - "commission": "10-30% recurring", "cookie_days": 30, "tag": "oxylabs", - "free_trial": "5K requests", "min_price": "$1.20/GB residential", - "note": "Enterprise-grade. Best for big data scraping."}, - {"name": "IPRoyal", "url": "https://dashboard.iproyal.com/signup?aff=pry", - "commission": "30% recurring lifetime", "cookie_days": 60, "tag": "iproyal", - "free_trial": "None", "min_price": "$0.04/GB residential", - "note": "Cheapest. Good for budget scraping."}, - {"name": "Webshare", "url": "https://proxy.webshare.io/register?aff=pry", - "commission": "30% recurring lifetime", "cookie_days": 60, "tag": "webshare", - "free_trial": "10 free proxies", "min_price": "$0.03/proxy/month", - "note": "Best free tier. Try before you buy."}, - {"name": "Proxy-Seller", "url": "https://proxy-seller.com/?partner=pry", - "commission": "30% recurring", "cookie_days": 60, "tag": "proxyseller", - "free_trial": "None", "min_price": "$1.40/proxy/month", - "note": "Cheap private proxies."}, - {"name": "NetNut", "url": "https://netnut.io/?aff=pry", - "commission": "Partner program", "cookie_days": 30, "tag": "netnut", - "free_trial": "7 days free", "min_price": "$0.20/GB", - "note": "Fast residential IPs from ISPs."}, - {"name": "ProxyMesh", "url": "https://proxymesh.com/?aff=pry", - "commission": "20% recurring", "cookie_days": 30, "tag": "proxymesh", - "free_trial": "None", "min_price": "$0.80/proxy", - "note": "Simple rotating proxies."}, - {"name": "Decodo (Smartproxy)", "url": "https://decodo.com/signup?aff=pry", - "commission": "20% recurring", "cookie_days": 60, "tag": "decodo", - "note": "Budget option from Smartproxy."}, - {"name": "PacketStream", "url": "https://packetstream.io/signup?aff=pry", - "commission": "20% revenue share", "cookie_days": 60, "tag": "packetstream", - "free_trial": "Pay-as-you-go", "min_price": "$0.05/GB", - "note": "Bandwidth-sharing residential proxies."}, + { + "name": "Bright Data", + "url": "https://brightdata.com/cp/start?aff_id=pry", + "commission": "$2-$50/signup, up to 10% recurring", + "cookie_days": 30, + "tag": "brightdata", + "free_trial": "$5 credit", + "min_price": "$0.10/GB residential", + "note": "Industry leader. Best for high-volume scraping.", + }, + { + "name": "Smartproxy", + "url": "https://dashboard.smartproxy.com/registration?aff_id=pry", + "commission": "20% recurring lifetime", + "cookie_days": 60, + "tag": "smartproxy", + "free_trial": "100MB free", + "min_price": "$0.10/GB residential", + "note": "Good balance of price and quality.", + }, + { + "name": "Oxylabs", + "url": "https://dashboard.oxylabs.io/?aff_id=pry", + "commission": "10-30% recurring", + "cookie_days": 30, + "tag": "oxylabs", + "free_trial": "5K requests", + "min_price": "$1.20/GB residential", + "note": "Enterprise-grade. Best for big data scraping.", + }, + { + "name": "IPRoyal", + "url": "https://dashboard.iproyal.com/signup?aff=pry", + "commission": "30% recurring lifetime", + "cookie_days": 60, + "tag": "iproyal", + "free_trial": "None", + "min_price": "$0.04/GB residential", + "note": "Cheapest. Good for budget scraping.", + }, + { + "name": "Webshare", + "url": "https://proxy.webshare.io/register?aff=pry", + "commission": "30% recurring lifetime", + "cookie_days": 60, + "tag": "webshare", + "free_trial": "10 free proxies", + "min_price": "$0.03/proxy/month", + "note": "Best free tier. Try before you buy.", + }, + { + "name": "Proxy-Seller", + "url": "https://proxy-seller.com/?partner=pry", + "commission": "30% recurring", + "cookie_days": 60, + "tag": "proxyseller", + "free_trial": "None", + "min_price": "$1.40/proxy/month", + "note": "Cheap private proxies.", + }, + { + "name": "NetNut", + "url": "https://netnut.io/?aff=pry", + "commission": "Partner program", + "cookie_days": 30, + "tag": "netnut", + "free_trial": "7 days free", + "min_price": "$0.20/GB", + "note": "Fast residential IPs from ISPs.", + }, + { + "name": "ProxyMesh", + "url": "https://proxymesh.com/?aff=pry", + "commission": "20% recurring", + "cookie_days": 30, + "tag": "proxymesh", + "free_trial": "None", + "min_price": "$0.80/proxy", + "note": "Simple rotating proxies.", + }, + { + "name": "Decodo (Smartproxy)", + "url": "https://decodo.com/signup?aff=pry", + "commission": "20% recurring", + "cookie_days": 60, + "tag": "decodo", + "note": "Budget option from Smartproxy.", + }, + { + "name": "PacketStream", + "url": "https://packetstream.io/signup?aff=pry", + "commission": "20% revenue share", + "cookie_days": 60, + "tag": "packetstream", + "free_trial": "Pay-as-you-go", + "min_price": "$0.05/GB", + "note": "Bandwidth-sharing residential proxies.", + }, ], "voice": [ - {"name": "ElevenLabs", "url": "https://elevenlabs.io/?ref=pry", - "commission": "22% recurring for 12 months", "cookie_days": 30, "tag": "elevenlabs"}, - {"name": "Murf AI", "url": "https://murf.ai/?ref=pry", - "commission": "30% recurring", "cookie_days": 30, "tag": "murf"}, - {"name": "Play.ht", "url": "https://play.ht/?ref=pry", - "commission": "20% recurring", "cookie_days": 30, "tag": "playht"}, + { + "name": "ElevenLabs", + "url": "https://elevenlabs.io/?ref=pry", + "commission": "22% recurring for 12 months", + "cookie_days": 30, + "tag": "elevenlabs", + }, + { + "name": "Murf AI", + "url": "https://murf.ai/?ref=pry", + "commission": "30% recurring", + "cookie_days": 30, + "tag": "murf", + }, + { + "name": "Play.ht", + "url": "https://play.ht/?ref=pry", + "commission": "20% recurring", + "cookie_days": 30, + "tag": "playht", + }, ], "media": [ - {"name": "RunwayML", "url": "https://runwayml.com/?ref=pry", - "commission": "20% first year", "cookie_days": 30, "tag": "runway"}, - {"name": "HeyGen", "url": "https://heygen.com/?ref=pry", - "commission": "20% recurring for 12 months", "cookie_days": 30, "tag": "heygen"}, - {"name": "Synthesia", "url": "https://synthesia.io/?ref=pry", - "commission": "20% recurring", "cookie_days": 30, "tag": "synthesia"}, - {"name": "Pika", "url": "https://pika.art/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "pika"}, - {"name": "Suno", "url": "https://suno.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "suno"}, - {"name": "ElevenLabs Image", "url": "https://elevenlabs.io/image?ref=pry", - "commission": "22% recurring", "cookie_days": 30, "tag": "elevenlabs-image"}, + { + "name": "RunwayML", + "url": "https://runwayml.com/?ref=pry", + "commission": "20% first year", + "cookie_days": 30, + "tag": "runway", + }, + { + "name": "HeyGen", + "url": "https://heygen.com/?ref=pry", + "commission": "20% recurring for 12 months", + "cookie_days": 30, + "tag": "heygen", + }, + { + "name": "Synthesia", + "url": "https://synthesia.io/?ref=pry", + "commission": "20% recurring", + "cookie_days": 30, + "tag": "synthesia", + }, + { + "name": "Pika", + "url": "https://pika.art/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "pika", + }, + { + "name": "Suno", + "url": "https://suno.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "suno", + }, + { + "name": "ElevenLabs Image", + "url": "https://elevenlabs.io/image?ref=pry", + "commission": "22% recurring", + "cookie_days": 30, + "tag": "elevenlabs-image", + }, ], "devtools": [ - {"name": "Cursor", "url": "https://cursor.com/?ref=pry", - "commission": "Varies", "cookie_days": 30, "tag": "cursor"}, - {"name": "v0.dev", "url": "https://v0.dev/?ref=pry", - "commission": "Varies", "cookie_days": 30, "tag": "v0"}, - {"name": "Bolt.new", "url": "https://bolt.new/?ref=pry", - "commission": "Varies", "cookie_days": 30, "tag": "bolt"}, - {"name": "Lovable", "url": "https://lovable.dev/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "lovable"}, - {"name": "Replit", "url": "https://replit.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "replit"}, + { + "name": "Cursor", + "url": "https://cursor.com/?ref=pry", + "commission": "Varies", + "cookie_days": 30, + "tag": "cursor", + }, + { + "name": "v0.dev", + "url": "https://v0.dev/?ref=pry", + "commission": "Varies", + "cookie_days": 30, + "tag": "v0", + }, + { + "name": "Bolt.new", + "url": "https://bolt.new/?ref=pry", + "commission": "Varies", + "cookie_days": 30, + "tag": "bolt", + }, + { + "name": "Lovable", + "url": "https://lovable.dev/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "lovable", + }, + { + "name": "Replit", + "url": "https://replit.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "replit", + }, ], "search": [ - {"name": "Algolia", "url": "https://algolia.com/?ref=pry", - "commission": "$200/paid signup", "cookie_days": 30, "tag": "algolia"}, - {"name": "Meilisearch Cloud", "url": "https://meilisearch.com/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "meilisearch"}, - {"name": "Typesense Cloud", "url": "https://typesense.org/?ref=pry", - "commission": "Usage share", "cookie_days": 30, "tag": "typesense"}, + { + "name": "Algolia", + "url": "https://algolia.com/?ref=pry", + "commission": "$200/paid signup", + "cookie_days": 30, + "tag": "algolia", + }, + { + "name": "Meilisearch Cloud", + "url": "https://meilisearch.com/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "meilisearch", + }, + { + "name": "Typesense Cloud", + "url": "https://typesense.org/?ref=pry", + "commission": "Usage share", + "cookie_days": 30, + "tag": "typesense", + }, ], "captcha": [ - {"name": "Capsolver", "url": "https://capsolver.com/?ref=pry", - "commission": "20% recurring lifetime", "cookie_days": 60, "tag": "capsolver"}, - {"name": "2Captcha", "url": "https://2captcha.com/?ref=pry", - "commission": "10% recurring lifetime", "cookie_days": 60, "tag": "2captcha"}, - {"name": "Anti-Captcha", "url": "https://anti-captcha.com/?ref=pry", - "commission": "10% recurring lifetime", "cookie_days": 60, "tag": "anticaptcha"}, - {"name": "CapMonster Cloud", "url": "https://capmonster.cloud/?ref=pry", - "commission": "10% recurring", "cookie_days": 60, "tag": "capmonster"}, - {"name": "NextCaptcha", "url": "https://nextcaptcha.com/?ref=pry", - "commission": "10% recurring", "cookie_days": 60, "tag": "nextcaptcha"}, + { + "name": "Capsolver", + "url": "https://capsolver.com/?ref=pry", + "commission": "20% recurring lifetime", + "cookie_days": 60, + "tag": "capsolver", + }, + { + "name": "2Captcha", + "url": "https://2captcha.com/?ref=pry", + "commission": "10% recurring lifetime", + "cookie_days": 60, + "tag": "2captcha", + }, + { + "name": "Anti-Captcha", + "url": "https://anti-captcha.com/?ref=pry", + "commission": "10% recurring lifetime", + "cookie_days": 60, + "tag": "anticaptcha", + }, + { + "name": "CapMonster Cloud", + "url": "https://capmonster.cloud/?ref=pry", + "commission": "10% recurring", + "cookie_days": 60, + "tag": "capmonster", + }, + { + "name": "NextCaptcha", + "url": "https://nextcaptcha.com/?ref=pry", + "commission": "10% recurring", + "cookie_days": 60, + "tag": "nextcaptcha", + }, ], "flaresolverr": [ - {"name": "FlareSolverr (Self-hosted)", "url": "https://github.com/FlareSolverr/FlareSolverr", - "commission": "Open source (no affiliate)", "cookie_days": 0, "tag": "flaresolverr", - "note": "Self-host for free; recommend on Hydra"}, + { + "name": "FlareSolverr (Self-hosted)", + "url": "https://github.com/FlareSolverr/FlareSolverr", + "commission": "Open source (no affiliate)", + "cookie_days": 0, + "tag": "flaresolverr", + "note": "Self-host for free; recommend on Hydra", + }, ], } @@ -235,17 +602,23 @@ class ReferralTracker: for line in path.read_text().splitlines(): if line.strip(): record = json.loads(line) - (self.clicks if path == self.clicks_file else self.conversions).append(record) + (self.clicks if path == self.clicks_file else self.conversions).append( + record + ) except (json.JSONDecodeError, OSError): pass - def record_click(self, provider_tag: str, url: str, source: str = "pry", - user_id: str = "") -> str: + def record_click( + self, provider_tag: str, url: str, source: str = "pry", user_id: str = "" + ) -> str: """Record a referral link click. Returns the tracking ID.""" click_id = uuid.uuid4().hex[:12] record = { - "id": click_id, "provider": provider_tag, "url": url, - "source": source, "user_id": user_id, + "id": click_id, + "provider": provider_tag, + "url": url, + "source": source, + "user_id": user_id, "timestamp": datetime.now(UTC).isoformat(), "converted": False, } @@ -258,8 +631,7 @@ class ReferralTracker: logger.info("referral_click", extra={"provider": provider_tag, "source": source}) return click_id - def record_conversion(self, click_id: str, revenue_usd: float = 0.0, - notes: str = "") -> bool: + def record_conversion(self, click_id: str, revenue_usd: float = 0.0, notes: str = "") -> bool: """Record a conversion for a click.""" for click in self.clicks: if click["id"] == click_id: @@ -267,8 +639,10 @@ class ReferralTracker: click["revenue_usd"] = revenue_usd click["conversion_notes"] = notes conversion = { - "click_id": click_id, "provider": click["provider"], - "revenue_usd": revenue_usd, "notes": notes, + "click_id": click_id, + "provider": click["provider"], + "revenue_usd": revenue_usd, + "notes": notes, "timestamp": datetime.now(UTC).isoformat(), } try: @@ -284,8 +658,9 @@ class ReferralTracker: """Get the provider catalog, optionally filtered by category.""" if category: return PROVIDER_CATALOG.get(category, []) - return [{"category": cat, "providers": providers} - for cat, providers in PROVIDER_CATALOG.items()] + return [ + {"category": cat, "providers": providers} for cat, providers in PROVIDER_CATALOG.items() + ] def get_stats(self, days_back: int = 30) -> dict[str, Any]: """Get referral statistics for the last N days.""" @@ -330,7 +705,8 @@ class ReferralTracker: def track_in_app_usage(self, provider_tag: str, revenue_usd: float) -> None: """Track revenue from in-app LLM provider usage (per-call).""" record = { - "type": "in_app_usage", "provider": provider_tag, + "type": "in_app_usage", + "provider": provider_tag, "revenue_usd": revenue_usd, "timestamp": datetime.now(UTC).isoformat(), } diff --git a/reports.py b/reports.py index b3d209e..f43e0d0 100644 --- a/reports.py +++ b/reports.py @@ -1,20 +1,19 @@ """Pry — Automated White-Label Report Generator. Generate branded PDF/HTML reports from scraped data for client delivery.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import logging import os import uuid from datetime import UTC, datetime, timedelta -from pathlib import Path from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) REPORTS_DIR = PRY_DATA_DIR / "reports" diff --git a/reports_real.py b/reports_real.py index da6ad69..967b242 100644 --- a/reports_real.py +++ b/reports_real.py @@ -6,13 +6,13 @@ """Pry — Real data-driven auto-reports with PDF generation.""" import logging -import os from datetime import UTC, datetime from html.parser import HTMLParser from pathlib import Path from typing import Any from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) REPORTS_DIR = PRY_DATA_DIR / "reports_real" @@ -84,7 +84,7 @@ class ReportGenerator: changes = data.get("changes", []) body = f"""

Competitive Analysis

-

Report generated: {datetime.now(UTC).strftime('%B %d, %Y')}

+

Report generated: {datetime.now(UTC).strftime("%B %d, %Y")}

Competitors Tracked ({len(competitors)})

@@ -95,10 +95,10 @@ class ReportGenerator: change_str = f"{change:+.1f}%" if change else "—" change_class = "up" if change > 0 else "down" if change < 0 else "stable" body += ( - f"" - f"" + f"" + f"" f"" - f"" + f"" ) body += "
NamePriceChangeLast Updated
{c.get('name','')}${c.get('price',0):.2f}
{c.get('name', '')}${c.get('price', 0):.2f}{change_str}{c.get('last_updated','')}
{c.get('last_updated', '')}
" @@ -115,7 +115,7 @@ class ReportGenerator: rises = sum(1 for p in products if (p.get("price_change") or 0) > 0) body = f"""

Price Monitoring Report

-

Generated: {datetime.now(UTC).strftime('%B %d, %Y')}

+

Generated: {datetime.now(UTC).strftime("%B %d, %Y")}

{len(products)}

Products

{drops}

Price Drops

@@ -128,9 +128,9 @@ class ReportGenerator: change_str = f"{pc:+.1f}%" if pc else "stable" change_class = "down" if pc < 0 else "up" if pc > 0 else "stable" body += ( - f"{p.get('name','')}" - f"${p.get('previous_price',0):.2f}" - f"${p.get('current_price',0):.2f}" + f"{p.get('name', '')}" + f"${p.get('previous_price', 0):.2f}" + f"${p.get('current_price', 0):.2f}" f"{change_str}" ) body += "" @@ -143,23 +143,21 @@ class ReportGenerator: desc = (seo.get("meta_description") or "N/A")[:200] body = f"""

SEO Audit Report

-

Generated: {datetime.now(UTC).strftime('%B %d, %Y')}

+

Generated: {datetime.now(UTC).strftime("%B %d, %Y")}

Current SEO Status

- - - - + + + +
Title{title_text}
Meta Description{desc}
Word Count{seo.get('word_count', 0):,}
Internal Links{seo.get('links_internal', 0)}
External Links{seo.get('links_external', 0)}
Schema Markup{'Yes' if seo.get('has_schema') else 'No'}
Word Count{seo.get("word_count", 0):,}
Internal Links{seo.get("links_internal", 0)}
External Links{seo.get("links_external", 0)}
Schema Markup{"Yes" if seo.get("has_schema") else "No"}
""" if changes: body += "

Recent SEO Changes

" return body @@ -167,16 +165,16 @@ class ReportGenerator: anomalies = data.get("anomalies", []) body = f"""

Anomaly Summary Report

-

Generated: {datetime.now(UTC).strftime('%B %d, %Y')}

+

Generated: {datetime.now(UTC).strftime("%B %d, %Y")}

Detected Anomalies ({len(anomalies)})

""" for a in anomalies: body += ( - f"" - f"" - f"" - f"" + f"" + f"" + f"" + f"" ) body += "
FieldTypeSeverityReason
{a.get('field','')}{a.get('type','')}{a.get('severity','')}{a.get('reason','')}
{a.get('field', '')}{a.get('type', '')}{a.get('severity', '')}{a.get('reason', '')}
" return body @@ -213,6 +211,7 @@ class ReportGenerator: def handle_endtag(self, tag: str) -> None: if tag == "tr" and self.current_row: import contextlib + with contextlib.suppress(Exception): self.story.append( Table( @@ -234,6 +233,7 @@ class ReportGenerator: self.current_row.append(safe) else: import contextlib + with contextlib.suppress(Exception): self.story.append(Paragraph(safe, self.styles["Normal"])) @@ -266,5 +266,5 @@ td {{ padding: 8px; border-bottom: 1px solid #eee; }}

{title}

{body} -
Generated by {company} on {datetime.now(UTC).strftime('%B %d, %Y at %H:%M UTC')}
+
Generated by {company} on {datetime.now(UTC).strftime("%B %d, %Y at %H:%M UTC")}
""" diff --git a/review.py b/review.py index f67c50b..81aa239 100644 --- a/review.py +++ b/review.py @@ -1,13 +1,11 @@ """Pry — Human-in-the-Loop Validation Workflow. Routes low-confidence extractions to human review before delivery.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging import os @@ -16,6 +14,8 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any, cast +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) REVIEW_DIR = PRY_DATA_DIR / "reviews" diff --git a/routers/auth.py b/routers/auth.py index 05a58d1..3e45907 100644 --- a/routers/auth.py +++ b/routers/auth.py @@ -16,6 +16,7 @@ Endpoints (6): Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper Licensed under MIT. See LICENSE. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -58,9 +59,7 @@ async def list_credentials_endpoint() -> dict[str, Any]: return {"success": True, "data": {"credentials": creds, "total": len(creds)}} -@router.delete( - "/v1/auth/credentials/{credential_id}", summary="Delete stored credentials" -) +@router.delete("/v1/auth/credentials/{credential_id}", summary="Delete stored credentials") async def delete_credentials_endpoint(credential_id: str) -> dict[str, Any]: """Delete stored credentials from the vault.""" from auth_connector import delete_credential diff --git a/schema_extraction.py b/schema_extraction.py index cc6e2ca..caae918 100644 --- a/schema_extraction.py +++ b/schema_extraction.py @@ -67,10 +67,10 @@ class SchemaExtractor: tree = lxml_html.fromstring(html) results: list[dict[str, Any]] = [] - for elem in tree.xpath('//*[@itemtype]'): + for elem in tree.xpath("//*[@itemtype]"): itemtype = elem.get("itemtype") or "" item: dict[str, Any] = {"@type": itemtype.split("/")[-1]} - for prop in elem.xpath('.//*[@itemprop]'): + for prop in elem.xpath(".//*[@itemprop]"): key = prop.get("itemprop") if not key: continue @@ -90,9 +90,9 @@ class SchemaExtractor: tree = lxml_html.fromstring(html) results: list[dict[str, Any]] = [] - for elem in tree.xpath('//*[@typeof]'): + for elem in tree.xpath("//*[@typeof]"): item: dict[str, Any] = {"@type": elem.get("typeof")} - for prop in elem.xpath('.//*[@property]'): + for prop in elem.xpath(".//*[@property]"): raw_key = prop.get("property") or "" key = raw_key.split(":")[-1] value = prop.get("content") or prop.text_content().strip() diff --git a/scraper.py b/scraper.py index d07e5b2..5273a3e 100644 --- a/scraper.py +++ b/scraper.py @@ -217,9 +217,12 @@ class PryScraper: if not self.playwright_available: try: import subprocess + r = subprocess.run( ["python3", "-m", "playwright", "install", "--dry-run", "chromium"], - capture_output=True, text=True, timeout=10 + capture_output=True, + text=True, + timeout=10, ) if "chromium" in r.stdout: self.playwright_available = True @@ -258,13 +261,11 @@ class PryScraper: if indicator.lower() in html_lower: return True # Check for very short / challenge-looking pages - if ( + return bool( len(html) < 2000 and "document" in html_lower and ("var " in html_lower or "function" in html_lower) - ): - return True - return False + ) def _quality_score(self, content: str) -> int: """Score content quality 0-100. Low score triggers retry with better tier.""" @@ -288,6 +289,7 @@ class PryScraper: # Use UltimateScraper for the 10-tier fetching if self._ultimate is None: from ultimate_scraper import UltimateScraper + self._ultimate = UltimateScraper() result = await self._ultimate.scrape(url, opts) @@ -563,5 +565,3 @@ class PryScraper: if urlparse(link).netloc == base_domain: links.add(link.split("#")[0].rstrip("/")) return sorted(links)[: opts.get("limit", 50)] - - diff --git a/secrets_backend.py b/secrets_backend.py index 28aa86a..a180ccb 100644 --- a/secrets_backend.py +++ b/secrets_backend.py @@ -23,6 +23,7 @@ The gopass lookup path is `pry/`. If you pass a name with leading Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper Licensed under MIT. See LICENSE. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -60,7 +61,9 @@ def _detect_backend() -> str: """Resolve which backend to use right now, based on env + availability.""" requested = os.getenv("PRY_SECRET_BACKEND", _DEFAULT_BACKEND).lower() if requested not in VALID_BACKENDS: - logger.warning("invalid_secret_backend", extra={"value": requested, "fallback": BACKEND_AUTO}) + logger.warning( + "invalid_secret_backend", extra={"value": requested, "fallback": BACKEND_AUTO} + ) return BACKEND_AUTO if requested == BACKEND_AUTO: return BACKEND_GOPASS if _gopass_available() else BACKEND_ENV @@ -163,7 +166,7 @@ def _get_from_file(name: str) -> str | None: """Read a secret from the .env file.""" values = _read_env_file() for key in (name, name.upper(), f"PRY_{name.upper()}", f"PRY_{name}"): - if key in values and values[key]: + if values.get(key): return values[key] return None @@ -205,7 +208,9 @@ def set_secret(name: str, value: str) -> bool: Returns True on success, False on failure. """ if not _gopass_available(): - logger.warning("set_secret_skipped", extra={"reason": "gopass not available", "secret_name": name}) + logger.warning( + "set_secret_skipped", extra={"reason": "gopass not available", "secret_name": name} + ) return False bin_path = shutil.which("gopass") path = _normalize_name(name) diff --git a/seo_monitor.py b/seo_monitor.py index 50b6d1e..fb36ab3 100644 --- a/seo_monitor.py +++ b/seo_monitor.py @@ -1,24 +1,23 @@ -import httpx + """Pry — SEO Content Change Monitor. Track competitor meta tags, titles, descriptions, headings, content changes.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import hashlib import json import logging -import os import re from contextlib import suppress from datetime import UTC, datetime -from pathlib import Path from typing import Any +import httpx + +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) SEO_DIR = PRY_DATA_DIR / "seo" @@ -76,6 +75,7 @@ async def analyze_seo(url: str) -> dict[str, Any]: if missing_critical: try: from llm_features import llm_seo_analyze + llm_enhancement = await llm_seo_analyze( url=url, html=resp.text[:6000], diff --git a/sessions.py b/sessions.py index 4a6e076..46a50f7 100644 --- a/sessions.py +++ b/sessions.py @@ -1,13 +1,11 @@ """Pry — persistent browser session management. Saves and restores Playwright browser contexts to disk.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import json import logging import os @@ -16,6 +14,8 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any, cast +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) SESSIONS_DIR = PRY_DATA_DIR / "sessions" diff --git a/signup_automator.py b/signup_automator.py index e38f1e3..d8f42bf 100644 --- a/signup_automator.py +++ b/signup_automator.py @@ -1,4 +1,4 @@ -import httpx + # SPDX-License-Identifier: BSL-1.1 # Copyright (c) 2026 Rug Munch Media LLC # @@ -17,6 +17,8 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any, ClassVar +import httpx + logger = logging.getLogger(__name__) PROFILES_DIR = Path(__file__).parent / "profiles" @@ -26,18 +28,132 @@ PROFILES_DIR.mkdir(parents=True, exist_ok=True) class ProfileGenerator: """Generate realistic human identities for account registration.""" - FIRST_NAMES: ClassVar[list[str]] = ["James", "Mary", "John", "Patricia", "Robert", "Jennifer", "Michael", "Linda", "William", "Elizabeth", - "David", "Barbara", "Richard", "Susan", "Joseph", "Jessica", "Thomas", "Sarah", "Christopher", "Karen"] - LAST_NAMES: ClassVar[list[str]] = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Rodriguez", "Martinez", - "Hernandez", "Lopez", "Gonzalez", "Wilson", "Anderson", "Thomas", "Taylor", "Moore", "Jackson", "Martin"] - DOMAINS: ClassVar[list[str]] = ["gmail.com", "yahoo.com", "outlook.com", "hotmail.com", "protonmail.com", "icloud.com", "aol.com", "mail.com"] - STREETS: ClassVar[list[str]] = ["Oak", "Maple", "Cedar", "Pine", "Elm", "Birch", "Walnut", "Cherry", "Ash", "Willow"] - STREET_TYPES: ClassVar[list[str]] = ["St", "Ave", "Rd", "Dr", "Ln", "Blvd", "Way", "Ct", "Pl", "Cir"] - CITIES: ClassVar[list[str]] = ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", "San Diego", - "Dallas", "Austin", "Portland", "Seattle", "Denver", "Boston", "Nashville"] - STATES: ClassVar[list[str]] = ["NY", "CA", "IL", "TX", "AZ", "PA", "TX", "CA", "TX", "TX", "OR", "WA", "CO", "MA", "TN"] - JOBS: ClassVar[list[str]] = ["Software Engineer", "Teacher", "Nurse", "Accountant", "Marketing Manager", "Data Analyst", "Graphic Designer", - "Project Manager", "Sales Associate", "Customer Support", "HR Coordinator", "Financial Analyst"] + FIRST_NAMES: ClassVar[list[str]] = [ + "James", + "Mary", + "John", + "Patricia", + "Robert", + "Jennifer", + "Michael", + "Linda", + "William", + "Elizabeth", + "David", + "Barbara", + "Richard", + "Susan", + "Joseph", + "Jessica", + "Thomas", + "Sarah", + "Christopher", + "Karen", + ] + LAST_NAMES: ClassVar[list[str]] = [ + "Smith", + "Johnson", + "Williams", + "Brown", + "Jones", + "Garcia", + "Miller", + "Davis", + "Rodriguez", + "Martinez", + "Hernandez", + "Lopez", + "Gonzalez", + "Wilson", + "Anderson", + "Thomas", + "Taylor", + "Moore", + "Jackson", + "Martin", + ] + DOMAINS: ClassVar[list[str]] = [ + "gmail.com", + "yahoo.com", + "outlook.com", + "hotmail.com", + "protonmail.com", + "icloud.com", + "aol.com", + "mail.com", + ] + STREETS: ClassVar[list[str]] = [ + "Oak", + "Maple", + "Cedar", + "Pine", + "Elm", + "Birch", + "Walnut", + "Cherry", + "Ash", + "Willow", + ] + STREET_TYPES: ClassVar[list[str]] = [ + "St", + "Ave", + "Rd", + "Dr", + "Ln", + "Blvd", + "Way", + "Ct", + "Pl", + "Cir", + ] + CITIES: ClassVar[list[str]] = [ + "New York", + "Los Angeles", + "Chicago", + "Houston", + "Phoenix", + "Philadelphia", + "San Antonio", + "San Diego", + "Dallas", + "Austin", + "Portland", + "Seattle", + "Denver", + "Boston", + "Nashville", + ] + STATES: ClassVar[list[str]] = [ + "NY", + "CA", + "IL", + "TX", + "AZ", + "PA", + "TX", + "CA", + "TX", + "TX", + "OR", + "WA", + "CO", + "MA", + "TN", + ] + JOBS: ClassVar[list[str]] = [ + "Software Engineer", + "Teacher", + "Nurse", + "Accountant", + "Marketing Manager", + "Data Analyst", + "Graphic Designer", + "Project Manager", + "Sales Associate", + "Customer Support", + "HR Coordinator", + "Financial Analyst", + ] def generate(self, locale: str = "US") -> dict[str, Any]: idx = random.randint(0, len(self.FIRST_NAMES) - 1) @@ -77,6 +193,7 @@ class ProfileGenerator: def _generate_id(self) -> str: import uuid + return uuid.uuid4().hex[:12] def save_profile(self, profile: dict[str, Any]) -> str: @@ -102,28 +219,44 @@ class EmailVerifier: async def create_temp_inbox(self) -> dict[str, Any]: """Create a temporary email inbox for receiving verification links.""" from client import get_client + client = await get_client() try: resp = await client.get("https://api.mail.tm/accounts", timeout=10) domains = resp.json().get("hydra:member", []) domain = domains[0]["domain"] if domains else "@mail.tm" import uuid + name = f"pry_{uuid.uuid4().hex[:8]}" pw = uuid.uuid4().hex[:16] - r = await client.post("https://api.mail.tm/accounts", - json={"address": f"{name}{domain}", "password": pw}, timeout=10) + r = await client.post( + "https://api.mail.tm/accounts", + json={"address": f"{name}{domain}", "password": pw}, + timeout=10, + ) if r.is_success: - token_r = await client.post("https://api.mail.tm/token", - json={"address": f"{name}{domain}", "password": pw}, timeout=10) + token_r = await client.post( + "https://api.mail.tm/token", + json={"address": f"{name}{domain}", "password": pw}, + timeout=10, + ) token = token_r.json().get("token", "") - return {"email": f"{name}{domain}", "password": pw, "token": token, "provider": "mail.tm"} + return { + "email": f"{name}{domain}", + "password": pw, + "token": token, + "provider": "mail.tm", + } except (httpx.HTTPError, httpx.RequestError) as e: logger.warning("temp_mail_failed", extra={"error": str(e)[:80]}) return {"email": "", "error": "Temp mail unavailable"} - async def check_verification_link(self, token: str, max_wait: int = 120, check_interval: int = 5) -> str | None: + async def check_verification_link( + self, token: str, max_wait: int = 120, check_interval: int = 5 + ) -> str | None: """Poll temp inbox for verification link.""" from client import get_client + client = await get_client() headers = {"Authorization": f"Bearer {token}"} for _ in range(max_wait // check_interval): @@ -133,12 +266,22 @@ class EmailVerifier: messages = r.json().get("hydra:member", []) for msg in messages: msg_id = msg.get("id", "") - mr = await client.get(f"https://api.mail.tm/messages/{msg_id}", headers=headers, timeout=10) + mr = await client.get( + f"https://api.mail.tm/messages/{msg_id}", headers=headers, timeout=10 + ) if mr.is_success: - html = mr.json().get("html", [""])[0] if isinstance(mr.json().get("html"), list) else str(mr.json()) + html = ( + mr.json().get("html", [""])[0] + if isinstance(mr.json().get("html"), list) + else str(mr.json()) + ) links = re.findall(r'https?://[^\s"<>]+', html) for link in links: - if "verify" in link.lower() or "confirm" in link.lower() or "activate" in link.lower(): + if ( + "verify" in link.lower() + or "confirm" in link.lower() + or "activate" in link.lower() + ): return link except (httpx.HTTPError, httpx.RequestError): pass @@ -149,30 +292,42 @@ class EmailVerifier: class SMSVerifier: """Handle SMS verification via Twilio or SMS-activate.""" - async def send_sms(self, phone: str, message: str, provider: str = "twilio", **creds: Any) -> dict[str, Any]: + async def send_sms( + self, phone: str, message: str, provider: str = "twilio", **creds: Any + ) -> dict[str, Any]: if provider == "twilio": return await self._twilio_send(phone, message, creds) return {"success": False, "error": f"Unknown SMS provider: {provider}"} async def _twilio_send(self, phone: str, message: str, creds: dict) -> dict[str, Any]: from client import get_client + client = await get_client() import base64 - auth = base64.b64encode(f"{creds.get('account_sid','')}:{creds.get('auth_token','')}".encode()).decode() + + auth = base64.b64encode( + f"{creds.get('account_sid', '')}:{creds.get('auth_token', '')}".encode() + ).decode() resp = await client.post( - f"https://api.twilio.com/2010-04-01/Accounts/{creds.get('account_sid','')}/Messages.json", + f"https://api.twilio.com/2010-04-01/Accounts/{creds.get('account_sid', '')}/Messages.json", data={"To": phone, "From": creds.get("from_number", ""), "Body": message}, - headers={"Authorization": f"Basic {auth}"}, timeout=15) + headers={"Authorization": f"Basic {auth}"}, + timeout=15, + ) return {"success": resp.is_success, "status": resp.status_code} async def request_activation_number(self, service: str, country: str = "us") -> dict[str, Any]: """Get a phone number for SMS activation via SMS-activate.org.""" from client import get_client + api_key = self._get_key() if not api_key: return {"success": False, "error": "SMS-activate API key required"} client = await get_client() - resp = await client.get(f"https://sms-activate.org/stubs/handler_api.php?api_key={api_key}&action=getNumber&service={service}&country={country}", timeout=15) + resp = await client.get( + f"https://sms-activate.org/stubs/handler_api.php?api_key={api_key}&action=getNumber&service={service}&country={country}", + timeout=15, + ) data = resp.text if "ACCESS_NUMBER" in data: parts = data.split(":") @@ -181,4 +336,5 @@ class SMSVerifier: def _get_key(self) -> str: import os + return os.getenv("PRY_SMS_ACTIVATE_KEY", "") diff --git a/stealth_engine.py b/stealth_engine.py index c7c8b11..c27d33e 100644 --- a/stealth_engine.py +++ b/stealth_engine.py @@ -24,7 +24,13 @@ class StealthEngine: def generate_all(self) -> str: """Generate complete stealth script with all fingerprint protections.""" - return self.webgl_noise() + self.canvas_noise() + self.audio_noise() + self.font_spoof() + self.webdriver_hide() + return ( + self.webgl_noise() + + self.canvas_noise() + + self.audio_noise() + + self.font_spoof() + + self.webdriver_hide() + ) def webgl_noise(self) -> str: """Add random noise to WebGL rendering to prevent GPU fingerprinting.""" @@ -87,7 +93,16 @@ class StealthEngine: def font_spoof(self) -> str: """Spoof system fonts to avoid font fingerprinting.""" - fonts = ["Arial", "Helvetica", "Times New Roman", "Courier New", "Verdana", "Georgia", "Palatino", "Garamond"] + fonts = [ + "Arial", + "Helvetica", + "Times New Roman", + "Courier New", + "Verdana", + "Georgia", + "Palatino", + "Garamond", + ] return f""" (() => {{ const fonts = {json.dumps(fonts)}; diff --git a/structure_monitor.py b/structure_monitor.py index 401ac22..317747d 100644 --- a/structure_monitor.py +++ b/structure_monitor.py @@ -1,26 +1,23 @@ -import httpx + """Pry — Page Structure Monitor. Track CSS selector validity over time. Alert before scrapers break.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import hashlib import json import logging -import os from contextlib import suppress from datetime import UTC, datetime -from pathlib import Path from typing import Any +import httpx from lxml import html as lxml_html from client import get_client +from paths import PRY_DATA_DIR logger = logging.getLogger(__name__) diff --git a/tasks.py b/tasks.py index aa18a04..ab1f5f1 100644 --- a/tasks.py +++ b/tasks.py @@ -1,29 +1,28 @@ """Pry — Background job system using asyncio (built-in) or Celery if available. Long-running jobs like crawls, monitors, and bulk imports should not block request threads.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import asyncio import json import logging -import os import uuid from collections.abc import Callable from datetime import UTC, datetime -from enum import Enum -from pathlib import Path +from enum import StrEnum from typing import Any +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) # Try Celery try: from celery import Celery + _has_celery = True except ImportError: _has_celery = False @@ -32,7 +31,7 @@ JOBS_DIR = PRY_DATA_DIR / "jobs" JOBS_DIR.mkdir(parents=True, exist_ok=True) -class JobStatus(str, Enum): +class JobStatus(StrEnum): PENDING = "pending" RUNNING = "running" COMPLETED = "completed" @@ -42,7 +41,10 @@ class JobStatus(str, Enum): class Job: """Represents a background job.""" - def __init__(self, job_id: str, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None): + + def __init__( + self, job_id: str, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None + ): self.id = job_id self.name = name self.func = func @@ -57,9 +59,14 @@ class Job: def to_dict(self) -> dict[str, Any]: return { - "id": self.id, "name": self.name, "status": self.status.value, - "created_at": self.created_at, "started_at": self.started_at, - "completed_at": self.completed_at, "result": self.result, "error": self.error, + "id": self.id, + "name": self.name, + "status": self.status.value, + "created_at": self.created_at, + "started_at": self.started_at, + "completed_at": self.completed_at, + "result": self.result, + "error": self.error, } @@ -75,7 +82,9 @@ class JobQueue: self._workers: list[asyncio.Task] = [] self._running = False - async def submit(self, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None) -> str: + async def submit( + self, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None + ) -> str: job_id = uuid.uuid4().hex[:12] job = Job(job_id, name, func, args, kwargs) self.jobs[job_id] = job @@ -86,7 +95,8 @@ class JobQueue: return job_id async def start(self) -> None: - if self._running or self._queue is None: return + if self._running or self._queue is None: + return self._running = True for i in range(self.max_workers): worker = asyncio.create_task(self._worker(f"worker-{i}")) @@ -109,7 +119,7 @@ class JobQueue: else: job.result = await asyncio.to_thread(job.func, *job.args, **job.kwargs) job.status = JobStatus.COMPLETED - except Exception as e: # noqa: BLE001 + except Exception as e: job.error = str(e)[:500] job.status = JobStatus.FAILED logger.exception("job_failed", extra={"job_id": job.id, "error": str(e)}) diff --git a/templates/test_live.py b/templates/test_live.py index 46c0b02..dd9b790 100644 --- a/templates/test_live.py +++ b/templates/test_live.py @@ -128,7 +128,7 @@ TEMPLATES = [ async def test_all(): failures = [] async with httpx.AsyncClient(timeout=30) as c: - for tid, site, cat in TEMPLATES: + for tid, site, _cat in TEMPLATES: print(f"Testing {tid}...", end=" ", flush=True) try: r = await c.post( @@ -146,7 +146,7 @@ async def test_all(): print(f"ERR: {e}") failures.append(tid) - print(f"\n{len([t for t in TEMPLATES])} tested, {len(failures)} failed") + print(f"\n{len(list(TEMPLATES))} tested, {len(failures)} failed") if failures: print("Failed:", ", ".join(failures)) return 1 if failures else 0 diff --git a/tests/conftest.py b/tests/conftest.py index 741c6e6..14f7f8c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,12 +56,33 @@ def sample_schema() -> dict: # # Production code is unaffected (this conftest only runs in tests). # If you actually want strict mode in tests, set PRY_LOG_STRICT_EXTRAS=1. -_RESERVED = frozenset({ - "name", "msg", "args", "levelname", "levelno", "pathname", "filename", - "module", "exc_info", "exc_text", "stack_info", "lineno", "funcName", - "created", "msecs", "relativeCreated", "thread", "threadName", - "processName", "process", "message", "asctime", "taskName", -}) +_RESERVED = frozenset( + { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", + "asctime", + "taskName", + } +) def _make_lenient_logrecord() -> None: diff --git a/tests/test_advanced_scraping.py b/tests/test_advanced_scraping.py index b1c928d..b5ec32d 100644 --- a/tests/test_advanced_scraping.py +++ b/tests/test_advanced_scraping.py @@ -107,12 +107,12 @@ def test_schema_extractor_jsonld_invalid() -> None: def test_schema_extractor_all() -> None: e = SchemaExtractor() html = ( - '' + "" '' '
' 'Widget' - '
' - '' + "
" + "" ) result = e.extract_all(html) assert result["count"] >= 2 diff --git a/tests/test_camoufox.py b/tests/test_camoufox.py index 1218593..2323869 100644 --- a/tests/test_camoufox.py +++ b/tests/test_camoufox.py @@ -24,7 +24,7 @@ def test_camoufox_init() -> None: def test_camoufox_config_completeness() -> None: - for name, config in CamoufoxBrowser.DEFAULT_CONFIGS.items(): + for _name, config in CamoufoxBrowser.DEFAULT_CONFIGS.items(): assert "headless" in config assert "os" in config assert "browser" in config diff --git a/tests/test_db.py b/tests/test_db.py index ead2715..f33572e 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -4,6 +4,7 @@ Uses a temp directory for PRY_DATA_DIR so we don't pollute the user's real data dir. Tests verify schema creation, basic CRUD, and the JSON store importer. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -13,8 +14,6 @@ from __future__ import annotations import importlib import json -import os -from pathlib import Path import pytest @@ -25,8 +24,9 @@ def temp_data_dir(monkeypatch, tmp_path): monkeypatch.setenv("PRY_DATA_DIR", str(tmp_path)) monkeypatch.setenv("PRY_DATABASE_URL", f"sqlite:///{tmp_path}/test.db") # Force re-import of paths and db - import paths import db + import paths + importlib.reload(paths) importlib.reload(db) # Reset the cached engine @@ -40,6 +40,7 @@ def temp_data_dir(monkeypatch, tmp_path): def test_get_engine_creates_db_file(temp_data_dir): import db + eng = db.get_engine() assert eng is not None # The DB file should exist now @@ -47,33 +48,58 @@ def test_get_engine_creates_db_file(temp_data_dir): def test_all_tables_created(temp_data_dir): - import db from sqlalchemy import inspect + + import db + db.get_engine() inspector = inspect(db.get_engine()) tables = inspector.get_table_names() # Should have all 24 model tables expected_tables = { - "quality_checks", "review_items", "intel_snapshots", "costing_entries", - "freshness_snapshots", "structure_snapshots", "seo_snapshots", "monitors", - "monitor_runs", "accounts", "browser_sessions", "reports", "training_datasets", - "pipelines", "pipeline_runs", "gdpr_requests", "agencies", "agency_clients", - "referral_clicks", "actors", "actor_runs", "llm_usage", "webhooks", "x402_receipts", + "quality_checks", + "review_items", + "intel_snapshots", + "costing_entries", + "freshness_snapshots", + "structure_snapshots", + "seo_snapshots", + "monitors", + "monitor_runs", + "accounts", + "browser_sessions", + "reports", + "training_datasets", + "pipelines", + "pipeline_runs", + "gdpr_requests", + "agencies", + "agency_clients", + "referral_clicks", + "actors", + "actor_runs", + "llm_usage", + "webhooks", + "x402_receipts", } assert expected_tables.issubset(set(tables)), f"missing: {expected_tables - set(tables)}" def test_session_scope_commits_on_success(temp_data_dir): import db + with db.session_scope() as s: - s.add(db.QualityCheck( - extraction_id="test-1", - url="https://example.com", - completeness=0.9, - )) + s.add( + db.QualityCheck( + extraction_id="test-1", + url="https://example.com", + completeness=0.9, + ) + ) # Verify it was committed with db.session_scope() as s: from sqlalchemy import select + result = s.execute(select(db.QualityCheck).where(db.QualityCheck.extraction_id == "test-1")) row = result.scalar_one_or_none() assert row is not None @@ -83,15 +109,19 @@ def test_session_scope_commits_on_success(temp_data_dir): def test_session_scope_rolls_back_on_error(temp_data_dir): import db + try: with db.session_scope() as s: - s.add(db.QualityCheck(extraction_id="test-2", url="https://example.com", completeness=0.5)) + s.add( + db.QualityCheck(extraction_id="test-2", url="https://example.com", completeness=0.5) + ) raise ValueError("intentional") except ValueError: pass # Verify it was NOT committed with db.session_scope() as s: from sqlalchemy import select + result = s.execute(select(db.QualityCheck).where(db.QualityCheck.extraction_id == "test-2")) assert result.scalar_one_or_none() is None @@ -102,31 +132,42 @@ def test_import_json_stores_reads_existing_data(temp_data_dir): intel_dir = temp_data_dir / "intel" intel_dir.mkdir(parents=True, exist_ok=True) (intel_dir / "snapshots.jsonl").write_text( # use the canonical filename - json.dumps({ - "competitor_id": "acme", - "competitor_name": "Acme Corp", - "url": "https://acme.com", - "fields": {"price": 99.99}, - }) + "\n" + - json.dumps({ - "competitor_id": "acme", - "competitor_name": "Acme Corp", - "url": "https://acme.com/pricing", - "fields": {"price": 89.99}, - }) + "\n" + json.dumps( + { + "competitor_id": "acme", + "competitor_name": "Acme Corp", + "url": "https://acme.com", + "fields": {"price": 99.99}, + } + ) + + "\n" + + json.dumps( + { + "competitor_id": "acme", + "competitor_name": "Acme Corp", + "url": "https://acme.com/pricing", + "fields": {"price": 89.99}, + } + ) + + "\n" ) monitor_dir = temp_data_dir / "monitors" monitor_dir.mkdir(parents=True, exist_ok=True) - (monitor_dir / "m1.json").write_text(json.dumps({ - "monitor_id": "m1", - "url": "https://example.com", - "name": "Example Monitor", - "schedule_cron": "0 * * * *", - "active": True, - })) + (monitor_dir / "m1.json").write_text( + json.dumps( + { + "monitor_id": "m1", + "url": "https://example.com", + "name": "Example Monitor", + "schedule_cron": "0 * * * *", + "active": True, + } + ) + ) import db + counts = db.import_json_stores(data_dir=temp_data_dir) assert counts.get("intel") == 2 assert counts.get("monitors") == 1 @@ -135,9 +176,16 @@ def test_import_json_stores_reads_existing_data(temp_data_dir): # Verify in SQL with db.session_scope() as s: from sqlalchemy import select - intel_rows = s.execute(select(db.IntelSnapshot).where(db.IntelSnapshot.competitor_id == "acme")).scalars().all() + + intel_rows = ( + s.execute(select(db.IntelSnapshot).where(db.IntelSnapshot.competitor_id == "acme")) + .scalars() + .all() + ) assert len(intel_rows) == 2 - monitor_row = s.execute(select(db.Monitor).where(db.Monitor.monitor_id == "m1")).scalar_one_or_none() + monitor_row = s.execute( + select(db.Monitor).where(db.Monitor.monitor_id == "m1") + ).scalar_one_or_none() assert monitor_row is not None # 'name' from JSON was renamed to 'monitor_name' in SQL assert monitor_row.monitor_name == "Example Monitor" @@ -145,6 +193,7 @@ def test_import_json_stores_reads_existing_data(temp_data_dir): def test_db_health_returns_dict(temp_data_dir): import db + health = db.db_health() assert isinstance(health, dict) assert "available" in health @@ -155,8 +204,10 @@ def test_db_health_returns_dict(temp_data_dir): def test_models_have_unique_constraints(temp_data_dir): """Tables that map to JSON stores with _id fields should have unique indexes so that re-importing the same data is idempotent (or at least detectable).""" - import db from sqlalchemy import inspect + + import db + db.get_engine() inspector = inspect(db.get_engine()) # monitor_id should be unique in monitors diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index a5af096..da1f00e 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -77,7 +77,10 @@ def test_db_is_available() -> None: def test_job_queue_submit() -> None: async def _test(): q = JobQueue() - async def my_task(x): return x * 2 + + async def my_task(x): + return x * 2 + job_id = await q.submit("double", my_task, args=(5,)) assert job_id in q.jobs # Run inline @@ -85,11 +88,12 @@ def test_job_queue_submit() -> None: await q._execute(job) assert job.status == JobStatus.COMPLETED assert job.result == 10 + asyncio.run(_test()) def test_job_listing() -> None: q = JobQueue() - job = q.jobs["test"] = type('J', (), {'to_dict': lambda self: {"id": "test", "name": "x"}})() + q.jobs["test"] = type("J", (), {"to_dict": lambda self: {"id": "test", "name": "x"}})() # Simple smoke test assert hasattr(q, "list_jobs") diff --git a/tests/test_llm_fallback.py b/tests/test_llm_fallback.py index 3b846cd..13a2adc 100644 --- a/tests/test_llm_fallback.py +++ b/tests/test_llm_fallback.py @@ -8,6 +8,7 @@ These tests verify that: The tests monkeypatch llm_features so we don't need a real LLM. """ + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -15,29 +16,30 @@ The tests monkeypatch llm_features so we don't need a real LLM. # Licensed under MIT. See LICENSE. from __future__ import annotations -import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest - # ── compliance.py ──────────────────────────────────────────────── + @pytest.mark.asyncio async def test_compliance_llm_fallback_merges_into_tos_result(): """When tos_result confidence is low, the LLM result should be merged in with the llm_enhanced flag set to True.""" from compliance import run_compliance_check - fake_llm = AsyncMock(return_value={ - "risk_level": "red", - "confidence": "high", - "risk_summary": "Strict scraping prohibition", - "recommendation": "Contact site owner", - "key_restrictions": ["no bots", "rate limited"], - "llm_provider": "openrouter", - "llm_cost_usd": 0.001, - }) + fake_llm = AsyncMock( + return_value={ + "risk_level": "red", + "confidence": "high", + "risk_summary": "Strict scraping prohibition", + "recommendation": "Contact site owner", + "key_restrictions": ["no bots", "rate limited"], + "llm_provider": "openrouter", + "llm_cost_usd": 0.001, + } + ) with patch("llm_features.llm_compliance_analyze", fake_llm): result = await run_compliance_check("https://example.com/") @@ -70,31 +72,36 @@ async def test_compliance_llm_fallback_preserves_result_on_error(): # ── seo_monitor.py ─────────────────────────────────────────────── + @pytest.mark.asyncio async def test_seo_llm_enhancement_fills_empty_critical_fields(): """When the regex pass leaves title/meta_description/h1 empty, the LLM should be invoked and its suggestions should fill the gaps.""" from seo_monitor import analyze_seo - fake_llm = AsyncMock(return_value={ - "title": "Pry - Web Intelligence", - "meta_description": "Open any website with Pry's free API", - "h1": "Web Scraping Made Simple", - "llm_provider": "ollama", - "llm_cost_usd": 0.0, - }) + fake_llm = AsyncMock( + return_value={ + "title": "Pry - Web Intelligence", + "meta_description": "Open any website with Pry's free API", + "h1": "Web Scraping Made Simple", + "llm_provider": "ollama", + "llm_cost_usd": 0.0, + } + ) # Patch the regex helpers to return empty - with patch("seo_monitor._get_title", return_value=""), \ - patch("seo_monitor._get_meta_content", return_value=""), \ - patch("seo_monitor._get_headings", return_value=[]), \ - patch("seo_monitor._count_words", return_value=100), \ - patch("seo_monitor._has_schema", return_value=False), \ - patch("seo_monitor._get_hreflangs", return_value=[]), \ - patch("seo_monitor._get_charset", return_value="utf-8"), \ - patch("seo_monitor._get_attr", return_value=""), \ - patch("seo_monitor._count_links", return_value=0), \ - patch("llm_features.llm_seo_analyze", fake_llm): + with ( + patch("seo_monitor._get_title", return_value=""), + patch("seo_monitor._get_meta_content", return_value=""), + patch("seo_monitor._get_headings", return_value=[]), + patch("seo_monitor._count_words", return_value=100), + patch("seo_monitor._has_schema", return_value=False), + patch("seo_monitor._get_hreflangs", return_value=[]), + patch("seo_monitor._get_charset", return_value="utf-8"), + patch("seo_monitor._get_attr", return_value=""), + patch("seo_monitor._count_links", return_value=0), + patch("llm_features.llm_seo_analyze", fake_llm), + ): # Mock the HTTP client to return a fake HTML response with patch("client.get_client") as mock_get_client: mock_resp = MagicMock() @@ -116,6 +123,7 @@ async def test_seo_llm_enhancement_fills_empty_critical_fields(): # ── reconciliation.py ──────────────────────────────────────────── + @pytest.mark.asyncio async def test_reconciliation_llm_enhance_function_exists(): """llm_enhance_reconciliation should be a callable that returns a dict.""" @@ -124,11 +132,13 @@ async def test_reconciliation_llm_enhance_function_exists(): fake_llm = AsyncMock(return_value={"is_same_entity": True}) with patch("llm_features.llm_entity_reconcile", fake_llm): - result = await llm_enhance_reconciliation([ - {"id": "1", "name": "Acme Widget", "confidence": 0.3, "group_id": "g1"}, - {"id": "2", "name": "Acme Widget Pro", "confidence": 0.4, "group_id": "g1"}, - {"id": "3", "name": "Other Product", "confidence": 0.95, "group_id": "g2"}, - ]) + result = await llm_enhance_reconciliation( + [ + {"id": "1", "name": "Acme Widget", "confidence": 0.3, "group_id": "g1"}, + {"id": "2", "name": "Acme Widget Pro", "confidence": 0.4, "group_id": "g1"}, + {"id": "3", "name": "Other Product", "confidence": 0.95, "group_id": "g2"}, + ] + ) assert result["llm_enhanced"] is True # Only the low-confidence group (g1) should have been sent to the LLM @@ -145,10 +155,12 @@ async def test_reconciliation_llm_enhance_handles_no_low_confidence(): fake_llm = AsyncMock(return_value={"is_same_entity": True}) with patch("llm_features.llm_entity_reconcile", fake_llm): - result = await llm_enhance_reconciliation([ - {"id": "1", "name": "A", "confidence": 0.9}, - {"id": "2", "name": "B", "confidence": 0.95}, - ]) + result = await llm_enhance_reconciliation( + [ + {"id": "1", "name": "A", "confidence": 0.9}, + {"id": "2", "name": "B", "confidence": 0.95}, + ] + ) assert not fake_llm.called assert result["llm_enhanced"] is False @@ -163,9 +175,11 @@ async def test_reconciliation_llm_enhance_handles_llm_error(): fake_llm = AsyncMock(side_effect=ConnectionError("timeout")) with patch("llm_features.llm_entity_reconcile", fake_llm): - result = await llm_enhance_reconciliation([ - {"id": "1", "name": "A", "confidence": 0.3, "group_id": "g1"}, - ]) + result = await llm_enhance_reconciliation( + [ + {"id": "1", "name": "A", "confidence": 0.3, "group_id": "g1"}, + ] + ) assert result["llm_enhanced"] is False assert "error" in result diff --git a/tests/test_llm_providers.py b/tests/test_llm_providers.py index df7ce1a..9066d37 100644 --- a/tests/test_llm_providers.py +++ b/tests/test_llm_providers.py @@ -30,15 +30,21 @@ def test_llm_registry_register() -> None: class FakeProvider(LLMProvider): name = "fake" - async def complete(self, *args, **kwargs): return LLMResponse(text="ok", model="fake", provider="fake") - async def embed(self, *args, **kwargs): return [0.1, 0.2] + + async def complete(self, *args, **kwargs): + return LLMResponse(text="ok", model="fake", provider="fake") + + async def embed(self, *args, **kwargs): + return [0.1, 0.2] r.register(FakeProvider()) assert "fake" in r.providers def test_llm_response_dataclass() -> None: - r = LLMResponse(text="hello", model="m", provider="p", input_tokens=10, output_tokens=5, cost_usd=0.001) + r = LLMResponse( + text="hello", model="m", provider="p", input_tokens=10, output_tokens=5, cost_usd=0.001 + ) assert r.text == "hello" assert r.cost_usd == 0.001 @@ -47,8 +53,13 @@ def test_provider_cost_estimation() -> None: class TestProvider(LLMProvider): cost_per_1k_input = 0.001 cost_per_1k_output = 0.002 - async def complete(self, *args, **kwargs): return LLMResponse(text="", model="t", provider="t") - async def embed(self, *args, **kwargs): return [] + + async def complete(self, *args, **kwargs): + return LLMResponse(text="", model="t", provider="t") + + async def embed(self, *args, **kwargs): + return [] + p = TestProvider() cost = p.estimate_cost(1000, 500) assert abs(cost - 0.002) < 0.0001 # 0.001 + 0.001 @@ -58,5 +69,6 @@ def test_referral_link_format() -> None: rc = ReferralConfig() for provider, link in rc.referral_links.items(): # FlareSolverr is open-source, no affiliate - if provider == "flaresolverr": continue + if provider == "flaresolverr": + continue assert "pry" in link, f"Link for {provider} missing referral: {link}" diff --git a/tests/test_logging_config.py b/tests/test_logging_config.py index 606c533..1b1a61b 100644 --- a/tests/test_logging_config.py +++ b/tests/test_logging_config.py @@ -1,4 +1,5 @@ """Tests for the logging_config module.""" + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -6,11 +7,8 @@ # Licensed under MIT. See LICENSE. from __future__ import annotations -import io import json import logging -import os -import sys import pytest @@ -89,6 +87,7 @@ def test_get_logger_returns_logger(): def test_setup_logging_without_structlog_falls_back(monkeypatch): """If structlog isn't available, setup_logging should still work.""" import builtins + real_import = builtins.__import__ def fake_import(name, *args, **kwargs): @@ -99,6 +98,7 @@ def test_setup_logging_without_structlog_falls_back(monkeypatch): monkeypatch.setattr(builtins, "__import__", fake_import) # Force a fresh import import importlib + importlib.reload(logging_config) try: logging_config.setup_logging() diff --git a/tests/test_marketplace.py b/tests/test_marketplace.py index dbbfc73..9449a40 100644 --- a/tests/test_marketplace.py +++ b/tests/test_marketplace.py @@ -35,8 +35,14 @@ def test_x402_middleware_signing() -> None: def test_actor_marketplace_create() -> None: m = ActorMarketplace() - actor = m.create("Test Actor", "Test description", template_id="amazon-product", - price_per_run=0.01, visibility=ActorVisibility.PRIVATE, tags=["e-commerce"]) + actor = m.create( + "Test Actor", + "Test description", + template_id="amazon-product", + price_per_run=0.01, + visibility=ActorVisibility.PRIVATE, + tags=["e-commerce"], + ) assert actor.name == "Test Actor" assert actor.template_id == "amazon-product" assert actor.price_per_run == 0.01 diff --git a/tests/test_mcp_production.py b/tests/test_mcp_production.py index 4dfc016..83cce7e 100644 --- a/tests/test_mcp_production.py +++ b/tests/test_mcp_production.py @@ -70,9 +70,7 @@ def test_resources() -> None: def test_fallback_initialize() -> None: server = make_fallback_server() register_all(server) - result = asyncio.run( - server.handle_request({"method": "initialize", "id": 1, "params": {}}) - ) + result = asyncio.run(server.handle_request({"method": "initialize", "id": 1, "params": {}})) assert "protocolVersion" in result["result"] assert result["result"]["serverInfo"]["name"] == "pry" assert result["result"]["serverInfo"]["version"] == "3.0.0" diff --git a/tests/test_referrals.py b/tests/test_referrals.py index 1888fce..b62ba39 100644 --- a/tests/test_referrals.py +++ b/tests/test_referrals.py @@ -27,15 +27,24 @@ def test_provider_format() -> None: for p in providers: assert "name" in p, f"Provider in {cat} missing name" assert "url" in p, f"Provider in {cat} missing url" - is_open_source = "open source" in str(p.get("commission", "")).lower() or "no affiliate" in str(p.get("note", "")).lower() + is_open_source = ( + "open source" in str(p.get("commission", "")).lower() + or "no affiliate" in str(p.get("note", "")).lower() + ) if is_open_source: continue - assert "pry" in p["url"].lower() or "ref" in p["url"].lower() or "affiliate" in p["url"].lower(), f"Provider {p['name']} URL missing referral: {p['url']}" + assert ( + "pry" in p["url"].lower() + or "ref" in p["url"].lower() + or "affiliate" in p["url"].lower() + ), f"Provider {p['name']} URL missing referral: {p['url']}" def test_referral_click() -> None: rt = ReferralTracker() - click_id = rt.record_click("openai", "https://platform.openai.com/signup?via=pry", source="test") + click_id = rt.record_click( + "openai", "https://platform.openai.com/signup?via=pry", source="test" + ) assert click_id is not None assert any(c["id"] == click_id for c in rt.clicks) diff --git a/tests/test_secrets_backend.py b/tests/test_secrets_backend.py index 0f59558..9d135b9 100644 --- a/tests/test_secrets_backend.py +++ b/tests/test_secrets_backend.py @@ -1,4 +1,5 @@ """Tests for secrets_backend module.""" + # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # @@ -8,17 +9,11 @@ from __future__ import annotations import importlib import os -from pathlib import Path from unittest.mock import patch -import pytest - import secrets_backend from secrets_backend import ( BACKEND_AUTO, - BACKEND_ENV, - BACKEND_FILE, - BACKEND_GOPASS, _normalize_name, backend_info, get_secret, diff --git a/tests/test_x402_mcp_spec.py b/tests/test_x402_mcp_spec.py index 62282f6..bb5df27 100644 --- a/tests/test_x402_mcp_spec.py +++ b/tests/test_x402_mcp_spec.py @@ -65,6 +65,7 @@ class _AlwaysVerifyRouter(FacilitatorRouter): "settled_at": "2026-01-01T00:00:00+00:00", } + # ── MCP spec compliance ── @@ -114,12 +115,14 @@ def test_initialize_response() -> None: """Test that initialize returns proper protocol version and capabilities.""" server = make_fallback_server() response = asyncio.run( - server.handle_request({ - "jsonrpc": "2.0", - "id": 1, - "method": "initialize", - "params": {"clientInfo": {"name": "test", "version": "1.0"}}, - }) + server.handle_request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": {"clientInfo": {"name": "test", "version": "1.0"}}, + } + ) ) assert response["jsonrpc"] == "2.0" assert response["id"] == 1 @@ -144,12 +147,14 @@ def test_tool_call_returns_content_array() -> None: server = make_fallback_server() register_all(server) response = asyncio.run( - server.handle_request({ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": {"name": "pry_x402_pricing", "arguments": {}}, - }) + server.handle_request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "pry_x402_pricing", "arguments": {}}, + } + ) ) assert "content" in response["result"] assert isinstance(response["result"]["content"], list) @@ -170,12 +175,14 @@ def test_resource_read_returns_contents() -> None: server = make_fallback_server() register_all(server) response = asyncio.run( - server.handle_request({ - "jsonrpc": "2.0", - "id": 1, - "method": "resources/read", - "params": {"uri": "pry://stats"}, - }) + server.handle_request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "resources/read", + "params": {"uri": "pry://stats"}, + } + ) ) assert "contents" in response["result"] @@ -194,12 +201,14 @@ def test_prompt_get_returns_messages() -> None: server = make_fallback_server() register_all(server) response = asyncio.run( - server.handle_request({ - "jsonrpc": "2.0", - "id": 1, - "method": "prompts/get", - "params": {"name": "research_company", "arguments": {"company_name": "Anthropic"}}, - }) + server.handle_request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "prompts/get", + "params": {"name": "research_company", "arguments": {"company_name": "Anthropic"}}, + } + ) ) assert "messages" in response["result"] assert len(response["result"]["messages"]) > 0 @@ -207,9 +216,7 @@ def test_prompt_get_returns_messages() -> None: def test_ping() -> None: server = make_fallback_server() - response = asyncio.run( - server.handle_request({"jsonrpc": "2.0", "id": 1, "method": "ping"}) - ) + response = asyncio.run(server.handle_request({"jsonrpc": "2.0", "id": 1, "method": "ping"})) assert response["result"] == {} @@ -272,9 +279,7 @@ def test_x402_signature_decoding() -> None: def test_x402_handler_create_payment_required() -> None: handler = X402Handler() - body, headers = handler.create_payment_required( - "scrape", "https://pry.dev/api/scrape" - ) + body, headers = handler.create_payment_required("scrape", "https://pry.dev/api/scrape") assert body["x402Version"] == 1 assert "PAYMENT-REQUIRED" in headers decoded = json.loads(base64.b64decode(headers["PAYMENT-REQUIRED"]).decode()) diff --git a/tls_fingerprint.py b/tls_fingerprint.py index 8948143..e339d0b 100644 --- a/tls_fingerprint.py +++ b/tls_fingerprint.py @@ -20,18 +20,47 @@ logger = logging.getLogger(__name__) # Check if curl_cffi is available try: from curl_cffi.requests import AsyncSession + _has_curl_cffi = True except ImportError: _has_curl_cffi = False # Browser fingerprints that curl_cffi can impersonate BROWSER_FINGERPRINTS = [ - "chrome", "chrome99", "chrome100", "chrome101", "chrome104", "chrome107", "chrome110", - "chrome116", "chrome119", "chrome120", "chrome123", "chrome124", "chrome131", "chrome133", - "chrome99_android", "chrome131_android", - "edge", "edge99", "edge101", "edge122", "edge127", - "firefox", "firefox109", "firefox117", "firefox120", "firefox123", "firefox124", "firefox132", - "safari", "safari15_3", "safari15_5", "safari16_5", "safari17_0", "safari17_2_ios", + "chrome", + "chrome99", + "chrome100", + "chrome101", + "chrome104", + "chrome107", + "chrome110", + "chrome116", + "chrome119", + "chrome120", + "chrome123", + "chrome124", + "chrome131", + "chrome133", + "chrome99_android", + "chrome131_android", + "edge", + "edge99", + "edge101", + "edge122", + "edge127", + "firefox", + "firefox109", + "firefox117", + "firefox120", + "firefox123", + "firefox124", + "firefox132", + "safari", + "safari15_3", + "safari15_5", + "safari16_5", + "safari17_0", + "safari17_2_ios", "tor", ] diff --git a/training_data.py b/training_data.py index 658ede4..17044ac 100644 --- a/training_data.py +++ b/training_data.py @@ -1,25 +1,23 @@ """Pry — AI Training Data Pipeline. Per-record provenance, license classifier, clean room export, compliance reports.""" -from paths import PRY_DATA_DIR # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import hashlib import json import logging -import os import re import uuid from collections import defaultdict from contextlib import suppress from datetime import UTC, datetime -from pathlib import Path from typing import Any, Literal +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) TRAINING_DIR = PRY_DATA_DIR / "training" diff --git a/ultimate_scraper.py b/ultimate_scraper.py index df0011c..c893a12 100644 --- a/ultimate_scraper.py +++ b/ultimate_scraper.py @@ -266,9 +266,7 @@ class UltimateScraper: try: import httpx - async with httpx.AsyncClient( - timeout=30, follow_redirects=True, proxy=proxy_url - ) as c: + async with httpx.AsyncClient(timeout=30, follow_redirects=True, proxy=proxy_url) as c: r = await c.get(url, headers=self._build_headers(url)) if r.is_success: return r.text @@ -283,9 +281,7 @@ class UltimateScraper: try: import httpx - async with httpx.AsyncClient( - timeout=45, follow_redirects=True, proxy=proxy_url - ) as c: + async with httpx.AsyncClient(timeout=45, follow_redirects=True, proxy=proxy_url) as c: r = await c.get(url, headers=self._build_headers(url)) if r.is_success: return r.text @@ -400,9 +396,7 @@ class UltimateScraper: headers["User-Agent"] = ( "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" ) - async with httpx.AsyncClient( - timeout=30, follow_redirects=True, proxy=proxy_url - ) as c: + async with httpx.AsyncClient(timeout=30, follow_redirects=True, proxy=proxy_url) as c: r = await c.get(url, headers=headers) if r.is_success: return r.text diff --git a/webhook_delivery.py b/webhook_delivery.py index b87e4e8..a88db08 100644 --- a/webhook_delivery.py +++ b/webhook_delivery.py @@ -1,14 +1,11 @@ -import httpx + """Pry — Webhook Delivery Service. Reliable webhook delivery with retries, signing (HMAC), and dead letter queue.""" -from paths import PRY_DATA_DIR - # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. - import asyncio import hashlib import hmac @@ -17,9 +14,12 @@ import logging import os import time from datetime import UTC, datetime -from pathlib import Path from typing import Any +import httpx + +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) WEBHOOK_DIR = PRY_DATA_DIR / "webhooks" @@ -32,7 +32,9 @@ class WebhookDelivery: DEFAULT_SIGNING_SECRET = "change-me-in-production" def __init__(self, signing_secret: str = ""): - self.signing_secret = signing_secret or os.getenv("PRY_WEBHOOK_SECRET", self.DEFAULT_SIGNING_SECRET) + self.signing_secret = signing_secret or os.getenv( + "PRY_WEBHOOK_SECRET", self.DEFAULT_SIGNING_SECRET + ) self.delivery_log = WEBHOOK_DIR / "deliveries.jsonl" self.dead_letter = WEBHOOK_DIR / "dead_letter.jsonl" self._load_log() @@ -61,6 +63,7 @@ class WebhookDelivery: ) -> dict[str, Any]: """Deliver a webhook with retries.""" from client import get_client + signature = self.sign_payload(payload) headers = { "Content-Type": "application/json", @@ -73,8 +76,11 @@ class WebhookDelivery: try: resp = await client.post(url, json=payload, headers=headers, timeout=10) record = { - "url": url, "event": event_type, "attempt": attempt, - "status": resp.status_code, "success": resp.is_success, + "url": url, + "event": event_type, + "attempt": attempt, + "status": resp.status_code, + "success": resp.is_success, "timestamp": datetime.now(UTC).isoformat(), } self._log(record) @@ -84,7 +90,12 @@ class WebhookDelivery: self._dead_letter(payload, url, f"HTTP {resp.status_code}") return {"success": False, "status": resp.status_code, "error": "Client error"} except (httpx.HTTPError, httpx.RequestError) as e: - record = {"url": url, "attempt": attempt, "error": str(e)[:200], "timestamp": datetime.now(UTC).isoformat()} + record = { + "url": url, + "attempt": attempt, + "error": str(e)[:200], + "timestamp": datetime.now(UTC).isoformat(), + } self._log(record) if attempt < max_retries: await asyncio.sleep(retry_delay * (2 ** (attempt - 1))) @@ -102,8 +113,17 @@ class WebhookDelivery: def _dead_letter(self, payload: dict, url: str, reason: str) -> None: try: with open(self.dead_letter, "a") as f: - f.write(json.dumps({"payload": payload, "url": url, "reason": reason, - "timestamp": datetime.now(UTC).isoformat()}) + "\n") + f.write( + json.dumps( + { + "payload": payload, + "url": url, + "reason": reason, + "timestamp": datetime.now(UTC).isoformat(), + } + ) + + "\n" + ) except OSError as e: logger.warning("dead_letter_write_failed", extra={"error": str(e)}) diff --git a/websocket_scraper.py b/websocket_scraper.py index a6c2fba..2e3adc9 100644 --- a/websocket_scraper.py +++ b/websocket_scraper.py @@ -56,10 +56,7 @@ class WebSocketScraper: subprotocols=protocols, ) as ws: start = time.time() - while ( - len(messages) < self.max_messages - and (time.time() - start) < self.timeout - ): + while len(messages) < self.max_messages and (time.time() - start) < self.timeout: try: msg = await asyncio.wait_for(ws.recv(), timeout=2) except TimeoutError: @@ -72,9 +69,7 @@ class WebSocketScraper: parsed: Any = json.loads(msg) except (json.JSONDecodeError, ValueError): parsed = msg - messages.append( - {"data": parsed, "raw": msg, "received_at": time.time()} - ) + messages.append({"data": parsed, "raw": msg, "received_at": time.time()}) except (json.JSONDecodeError, ValueError) as e: return {"success": False, "error": str(e)[:300]} diff --git a/x402.py b/x402.py index ae529aa..80b926d 100644 --- a/x402.py +++ b/x402.py @@ -19,7 +19,6 @@ Standards compliance: # Licensed under MIT. See LICENSE. from __future__ import annotations -from paths import PRY_DATA_DIR import base64 import json @@ -30,11 +29,12 @@ import uuid from dataclasses import dataclass, field from datetime import UTC, datetime from enum import StrEnum -from pathlib import Path from typing import Any, cast import httpx +from paths import PRY_DATA_DIR + logger = logging.getLogger(__name__) X402_DIR = PRY_DATA_DIR / "x402" @@ -103,8 +103,11 @@ class X402Asset(StrEnum): # PRY_X402_WALLET=0x... # one-off override (CI, dev) try: from secrets_backend import get_secret as _gs + X402_WALLET = _gs("x402_wallet") or os.getenv("PRY_X402_WALLET", "pry-default-wallet") - X402_FACILITATOR_URL = _gs("x402_facilitator") or os.getenv("PRY_X402_FACILITATOR", "https://x402.org/facilitator") + X402_FACILITATOR_URL = _gs("x402_facilitator") or os.getenv( + "PRY_X402_FACILITATOR", "https://x402.org/facilitator" + ) except ImportError: X402_WALLET = os.getenv("PRY_X402_WALLET", "pry-default-wallet") X402_FACILITATOR_URL = os.getenv("PRY_X402_FACILITATOR", "https://x402.org/facilitator") @@ -260,30 +263,34 @@ def build_payment_required( if not _asset_on_network(asset, network): continue amount = _to_atomic(price_info["price_usd"], asset) - accepts.append({ + accepts.append( + { + "scheme": X402Scheme.EXACT.value, + "network": network, + "maxAmountRequired": str(amount), + "resource": resource_url, + "description": description or price_info.get("description", operation), + "mimeType": mime_type, + "payToAddress": X402_WALLET, + "asset": asset, + "extra": {"name": asset, "version": "2", "operation": operation}, + } + ) + if not accepts: + # Final fallback so we never return an empty accepts array + accepts.append( + { "scheme": X402Scheme.EXACT.value, - "network": network, - "maxAmountRequired": str(amount), + "network": X402Network.BASE.value, + "maxAmountRequired": str(amount_atomic), "resource": resource_url, "description": description or price_info.get("description", operation), "mimeType": mime_type, "payToAddress": X402_WALLET, - "asset": asset, - "extra": {"name": asset, "version": "2", "operation": operation}, - }) - if not accepts: - # Final fallback so we never return an empty accepts array - accepts.append({ - "scheme": X402Scheme.EXACT.value, - "network": X402Network.BASE.value, - "maxAmountRequired": str(amount_atomic), - "resource": resource_url, - "description": description or price_info.get("description", operation), - "mimeType": mime_type, - "payToAddress": X402_WALLET, - "asset": X402_DEFAULT_ASSET, - "extra": {"name": X402_DEFAULT_ASSET, "version": "2", "operation": operation}, - }) + "asset": X402_DEFAULT_ASSET, + "extra": {"name": X402_DEFAULT_ASSET, "version": "2", "operation": operation}, + } + ) return { "x402Version": 1, @@ -397,7 +404,9 @@ class FacilitatorRouter: response.raise_for_status() data = response.json() self._mark_success(facilitator) - verified = bool(data.get("isValid") or data.get("verified") or data.get("success")) + verified = bool( + data.get("isValid") or data.get("verified") or data.get("success") + ) return { "verified": verified, "reason": data.get("error", data.get("reason", "")) if not verified else "", @@ -458,7 +467,9 @@ class FacilitatorRouter: response.raise_for_status() data = response.json() self._mark_success(facilitator) - return self._normalize_settlement(data, payment_id, tx_hash, network, facilitator.name) + return self._normalize_settlement( + data, payment_id, tx_hash, network, facilitator.name + ) except httpx.HTTPStatusError as e: last_error = f"{facilitator.name}: HTTP {e.response.status_code}" self._mark_failure(facilitator) @@ -819,7 +830,9 @@ async def create_batch_payment(operations: list[str]) -> dict[str, Any]: } -async def verify_batch_payment(batch_id: str, tx_hash: str, network: str = "", asset: str = "") -> dict[str, Any]: +async def verify_batch_payment( + batch_id: str, tx_hash: str, network: str = "", asset: str = "" +) -> dict[str, Any]: """Verify a batch payment on-chain and mark it paid if valid.""" with _batch_lock: batch = _batch_payments.get(batch_id) @@ -853,6 +866,8 @@ def is_batch_paid(batch_id: str) -> bool: """Check whether a batch payment has been verified.""" batch = _batch_payments.get(batch_id) return bool(batch and batch.get("paid")) + + # These return the pre-spec response format. Prefer the spec-compliant methods # above in new code. diff --git a/x402_middleware.py b/x402_middleware.py index 28bc59c..06afc34 100644 --- a/x402_middleware.py +++ b/x402_middleware.py @@ -74,9 +74,7 @@ class X402Middleware: return path = scope.get("path", "") - if path in self.free_paths or not any( - path.startswith(p) for p in PAID_ENDPOINTS - ): + if path in self.free_paths or not any(path.startswith(p) for p in PAID_ENDPOINTS): await self.app(scope, receive, send) return @@ -93,6 +91,7 @@ class X402Middleware: # Check pre-verified batch payment if batch_id_header: from x402 import is_batch_paid + if is_batch_paid(batch_id_header): await self.app(scope, receive, send) return @@ -113,9 +112,7 @@ class X402Middleware: settlement = await self.handler.settle_payment( payment_id, tx_hash, network=network, asset=asset, amount_usd=amount_usd ) - response_b64 = base64.b64encode( - json.dumps(settlement).encode() - ).decode() + response_b64 = base64.b64encode(json.dumps(settlement).encode()).decode() self.payments[payment_id] = { "payment_id": payment_id, "tx_hash": tx_hash, @@ -136,16 +133,12 @@ class X402Middleware: except (ValueError, KeyError, TypeError) as e: logger.warning("x402_payment_decode_failed err=%s", e) - operation = next( - (op for ep, op in PAID_ENDPOINTS.items() if path.startswith(ep)), None - ) + operation = next((op for ep, op in PAID_ENDPOINTS.items() if path.startswith(ep)), None) if not operation or operation not in X402_PRICING: await self.app(scope, receive, send) return - body, x402_headers = self.handler.create_payment_required( - operation, resource=path - ) + body, x402_headers = self.handler.create_payment_required(operation, resource=path) body_bytes = json.dumps(body).encode() response_headers = [ @@ -155,9 +148,11 @@ class X402Middleware: for k, v in x402_headers.items(): response_headers.append((k.lower().encode(), v.encode())) - await send({ - "type": "http.response.start", - "status": 402, - "headers": response_headers, - }) + await send( + { + "type": "http.response.start", + "status": 402, + "headers": response_headers, + } + ) await send({"type": "http.response.body", "body": body_bytes})