refactor(settings): unify config.py, mconfig.py, and settings.py into single PrySettings
- Merge all configuration into settings.py with PRY_* env var prefix. - Add legacy env aliases (PROXY_URL, TOR_ENABLED, MAX_RETRIES, etc.) for migration. - Load and persist runtime overrides from JSON config file (default /app/config.json). - Update scraper.py, deps.py, routers/config.py to use unified settings. - Add resolved_proxy_chain and resolved_proxy_url properties. - Replace tests/test_mconfig.py with tests/test_settings.py (9 tests). - Update .env.example with new PRY_* options. - All 500 tests pass; ruff clean.
This commit is contained in:
parent
ecb05bbf49
commit
1f9e71d294
9 changed files with 424 additions and 316 deletions
23
.env.example
23
.env.example
|
|
@ -69,3 +69,26 @@ PRY_FLARESOLVERR_URL=http://flaresolverr:8191/v1
|
|||
|
||||
# ── Redis ──
|
||||
# PRY_REDIS_URL=redis://localhost:6379/0
|
||||
|
||||
# ── Database ──
|
||||
# PRY_DATABASE_URL=postgresql+asyncpg://pry:pry@localhost/pry
|
||||
|
||||
# ── Stealth ──
|
||||
# PRY_STEALTH_ENABLED=true
|
||||
# PRY_RANDOM_USER_AGENT=true
|
||||
# PRY_WEBDRIVER_OVERRIDE=true
|
||||
# PRY_CANVAS_NOISE=true
|
||||
# PRY_WEBRTC_DISABLE=true
|
||||
# PRY_GEOLOCATION_SPOOF=true
|
||||
# PRY_MIN_DELAY_MS=500
|
||||
# PRY_MAX_DELAY_MS=3000
|
||||
|
||||
# ── Output ──
|
||||
# PRY_DEFAULT_FORMAT=markdown
|
||||
# PRY_MAX_CHARS=100000
|
||||
# PRY_INCLUDE_LINKS=true
|
||||
|
||||
# ── x402 / MCP ──
|
||||
# PRY_X402_ENABLED=false
|
||||
# PRY_X402_PAY_TO=
|
||||
# PRY_MCP_ENABLED=true
|
||||
|
|
|
|||
80
config.py
80
config.py
|
|
@ -1,80 +0,0 @@
|
|||
"""PryScraper — global configuration.
|
||||
|
||||
Loads settings from environment variables (via Pydantic Settings).
|
||||
All secrets come from gopass, never from .env files committed to git.
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""PryScraper application settings."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# Core
|
||||
app_name: str = "PryScraper"
|
||||
app_version: str = "3.0.0"
|
||||
environment: Literal["dev", "staging", "prod"] = "dev"
|
||||
log_level: str = "INFO"
|
||||
port: int = 8002
|
||||
|
||||
# Database
|
||||
database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry"
|
||||
|
||||
# Cache
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
|
||||
# LLM Providers
|
||||
openai_api_key: str = ""
|
||||
anthropic_api_key: str = ""
|
||||
cohere_api_key: str = ""
|
||||
ollama_url: str = "http://localhost:11434"
|
||||
|
||||
# Stealth
|
||||
stealth_enabled: bool = True
|
||||
random_user_agent: bool = True
|
||||
min_delay_ms: int = 500
|
||||
max_delay_ms: int = 3000
|
||||
|
||||
# Rate limiting
|
||||
rate_limit_per_domain: int = 10 # requests per second
|
||||
rate_limit_per_ip: int = 60
|
||||
|
||||
# x402 Payment
|
||||
x402_enabled: bool = False
|
||||
x402_pay_to: str = ""
|
||||
|
||||
# MCP
|
||||
mcp_enabled: bool = True
|
||||
mcp_tools_count: int = 8
|
||||
|
||||
# Storage
|
||||
screenshot_dir: Path = Path("/tmp/pry-screenshots") # nosec B108
|
||||
cache_ttl_seconds: int = 3600
|
||||
|
||||
|
||||
_settings: Settings | None = None
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
"""Get cached settings instance."""
|
||||
global _settings
|
||||
if _settings is None:
|
||||
_settings = Settings()
|
||||
return _settings
|
||||
8
deps.py
8
deps.py
|
|
@ -6,8 +6,6 @@ direct access to shared state.
|
|||
|
||||
Usage:
|
||||
from deps import scraper, cache, automator, ...
|
||||
|
||||
Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
|
@ -19,18 +17,16 @@ from automator import PryAutomator
|
|||
from cache import ResponseCache
|
||||
from extractor import SchemaExtractor
|
||||
from jobqueue import JobQueue
|
||||
from mconfig import PryConfig
|
||||
from parser import DocumentParser
|
||||
from ratelimit import RateLimiter
|
||||
from scraper import PryScraper
|
||||
|
||||
config = PryConfig()
|
||||
from settings import settings
|
||||
|
||||
scraper = PryScraper()
|
||||
automator = PryAutomator()
|
||||
parser = DocumentParser()
|
||||
extractor = SchemaExtractor()
|
||||
cache = ResponseCache(capacity=1000)
|
||||
ratelimiter = RateLimiter(default_rpm=120, burst=200)
|
||||
ratelimiter = RateLimiter(default_rpm=settings.rate_limit_rpm, burst=200)
|
||||
queue = JobQueue()
|
||||
advanced = PryAdvanced(cache=cache)
|
||||
|
|
|
|||
133
mconfig.py
133
mconfig.py
|
|
@ -1,133 +0,0 @@
|
|||
"""Pry Config — runtime configuration for proxy, Tor, VPN, and anti-detection.
|
||||
Users can configure at startup (env vars) or at runtime (API calls)."""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
CONFIG_FILE = "/app/config.json"
|
||||
DEFAULT_CONFIG = {
|
||||
"proxy": {"enabled": False, "type": "", "url": "", "username": "", "password": ""},
|
||||
"tor": {"enabled": False, "socks5_host": "tor", "socks5_port": 9050, "control_port": 9051},
|
||||
"rotation": {
|
||||
"user_agent": "rotate",
|
||||
"ip": "off",
|
||||
"timing": "random",
|
||||
"delay_range": [0.5, 3.0],
|
||||
},
|
||||
"stealth": {
|
||||
"webdriver_override": True,
|
||||
"canvas_noise": True,
|
||||
"webrtc_disable": True,
|
||||
"geolocation_spoof": True,
|
||||
},
|
||||
"retry": {"max_attempts": 3, "min_quality": 20, "backoff": "exponential"},
|
||||
"output": {"default_format": "markdown", "max_chars": 100000, "include_links": True},
|
||||
"rate_limit": {"rpm": 120, "burst": 200},
|
||||
}
|
||||
|
||||
|
||||
class PryConfig:
|
||||
"""Configuration manager — loads from env vars, config file, and API overrides."""
|
||||
|
||||
def __init__(self):
|
||||
self.config = json.loads(json.dumps(DEFAULT_CONFIG)) # Deep copy
|
||||
self._load_env()
|
||||
self._load_file()
|
||||
self._resolve_proxy_chain()
|
||||
|
||||
def _load_env(self):
|
||||
"""Environment variables override defaults."""
|
||||
if os.getenv("TOR_ENABLED", "").lower() in ("1", "true", "yes"):
|
||||
self.config["tor"]["enabled"] = True
|
||||
if os.getenv("TOR_SOCKS5_HOST"):
|
||||
self.config["tor"]["socks5_host"] = os.getenv("TOR_SOCKS5_HOST")
|
||||
if os.getenv("TOR_SOCKS5_PORT"):
|
||||
self.config["tor"]["socks5_port"] = int(os.getenv("TOR_SOCKS5_PORT"))
|
||||
if os.getenv("PROXY_URL"):
|
||||
self.config["proxy"]["enabled"] = True
|
||||
self.config["proxy"]["url"] = os.getenv("PROXY_URL")
|
||||
self.config["proxy"]["type"] = os.getenv("PROXY_TYPE", "http")
|
||||
if os.getenv("PROXY_USERNAME"):
|
||||
self.config["proxy"]["username"] = os.getenv("PROXY_USERNAME")
|
||||
if os.getenv("PROXY_PASSWORD"):
|
||||
self.config["proxy"]["password"] = os.getenv("PROXY_PASSWORD")
|
||||
if os.getenv("IP_ROTATION"):
|
||||
self.config["rotation"]["ip"] = os.getenv("IP_ROTATION")
|
||||
if os.getenv("MAX_RETRIES"):
|
||||
self.config["retry"]["max_attempts"] = int(os.getenv("MAX_RETRIES"))
|
||||
if os.getenv("MIN_QUALITY"):
|
||||
self.config["retry"]["min_quality"] = int(os.getenv("MIN_QUALITY"))
|
||||
if os.getenv("RATE_LIMIT_RPM"):
|
||||
self.config["rate_limit"]["rpm"] = int(os.getenv("RATE_LIMIT_RPM"))
|
||||
|
||||
def _load_file(self):
|
||||
if os.path.exists(CONFIG_FILE):
|
||||
try:
|
||||
with open(CONFIG_FILE) as f:
|
||||
user_config = json.load(f)
|
||||
self._deep_merge(self.config, user_config)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _deep_merge(self, base: dict, override: dict):
|
||||
for key, value in override.items():
|
||||
if key in base and isinstance(base[key], dict) and isinstance(value, dict):
|
||||
self._deep_merge(base[key], value)
|
||||
else:
|
||||
base[key] = value
|
||||
|
||||
def _resolve_proxy_chain(self):
|
||||
"""Build the proxy chain: Tor -> User Proxy -> Direct"""
|
||||
proxies = []
|
||||
if self.config["tor"]["enabled"]:
|
||||
proxies.append(
|
||||
f"socks5://{self.config['tor']['socks5_host']}:{self.config['tor']['socks5_port']}"
|
||||
)
|
||||
if self.config["proxy"]["enabled"]:
|
||||
url = self.config["proxy"]["url"]
|
||||
if self.config["proxy"]["username"]:
|
||||
netloc = url.split("://")[1] if "://" in url else url
|
||||
proxies.append(
|
||||
f"{self.config['proxy']['type']}://{self.config['proxy']['username']}:{self.config['proxy']['password']}@{netloc}"
|
||||
)
|
||||
else:
|
||||
proxies.append(url)
|
||||
self.config["_proxy_chain"] = proxies
|
||||
self.config["_proxy_url"] = proxies[0] if proxies else None
|
||||
|
||||
def get_proxy_url(self) -> str | None:
|
||||
return self.config.get("_proxy_url")
|
||||
|
||||
def get_proxy_chain(self) -> list:
|
||||
return self.config.get("_proxy_chain", [])
|
||||
|
||||
def get(self, key: str, default=None):
|
||||
keys = key.split(".")
|
||||
val = self.config
|
||||
for k in keys:
|
||||
if isinstance(val, dict):
|
||||
val = val.get(k)
|
||||
else:
|
||||
return default
|
||||
if val is None:
|
||||
return default
|
||||
return val if val is not None else default
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {k: v for k, v in self.config.items() if not k.startswith("_")}
|
||||
|
||||
def update(self, updates: dict) -> dict:
|
||||
self._deep_merge(self.config, updates)
|
||||
self._resolve_proxy_chain()
|
||||
try:
|
||||
with open(CONFIG_FILE, "w") as f:
|
||||
json.dump(self.to_dict(), f, indent=2)
|
||||
except OSError:
|
||||
pass
|
||||
return {"status": "ok", "config": self.to_dict()}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
"""Pry — Config router (remaining api.py routes).
|
||||
"""Pry — Config router (runtime configuration management).
|
||||
|
||||
Auto-extracted from api.py during the router-split refactor.
|
||||
Provides endpoints to read and update the unified Pry settings object.
|
||||
Runtime changes are persisted to the configured JSON file.
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
|
@ -13,37 +14,33 @@ from typing import Any
|
|||
|
||||
from fastapi import APIRouter, Body
|
||||
|
||||
from mconfig import PryConfig
|
||||
from settings import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(tags=["Config"])
|
||||
|
||||
|
||||
@router.get("/v1/config", tags=["Config"], summary="Get current Pry configuration")
|
||||
async def get_config() -> dict[str, Any]:
|
||||
"""Get current Pry configuration."""
|
||||
return {"success": True, "data": config.to_dict()}
|
||||
return {"success": True, "data": settings.to_api_dict()}
|
||||
|
||||
|
||||
@router.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime")
|
||||
async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||
"""Update Pry configuration at runtime."""
|
||||
result = config.update(updates)
|
||||
result = settings.update(updates)
|
||||
return {"success": True, "data": result}
|
||||
|
||||
|
||||
@router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests")
|
||||
async def enable_tor() -> dict[str, Any]:
|
||||
"""Enable Tor routing for all requests."""
|
||||
result = config.update(
|
||||
{"tor": {"enabled": True}, "proxy": {"enabled": True, "url": "socks5://tor:9050"}}
|
||||
)
|
||||
result = settings.enable_tor()
|
||||
resp_data = {
|
||||
"success": True,
|
||||
"data": result,
|
||||
"note": "Run 'docker compose --profile tor up -d' to start Tor container",
|
||||
}
|
||||
return resp_data
|
||||
|
||||
|
||||
config = PryConfig()
|
||||
|
|
|
|||
11
scraper.py
11
scraper.py
|
|
@ -19,10 +19,7 @@ import httpx
|
|||
import trafilatura
|
||||
from trafilatura.settings import use_config
|
||||
|
||||
from mconfig import PryConfig
|
||||
|
||||
FLARESOLVERR_URL = "http://flaresolverr:8191/v1"
|
||||
config = PryConfig()
|
||||
from settings import settings
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
|
|
@ -415,7 +412,7 @@ class PryScraper:
|
|||
return result
|
||||
|
||||
async def _fetch_direct(self, url: str, headers: dict, timeout: int) -> str:
|
||||
proxy_url = config.get_proxy_url()
|
||||
proxy_url = settings.resolved_proxy_url
|
||||
async with httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(timeout),
|
||||
follow_redirects=True,
|
||||
|
|
@ -424,7 +421,7 @@ class PryScraper:
|
|||
cookies=httpx.Cookies(),
|
||||
proxy=proxy_url,
|
||||
) as client:
|
||||
delay = random.uniform(*config.get("rotation.delay_range", [0.5, 3.0]))
|
||||
delay = random.uniform(*settings.rotation_delay_range)
|
||||
await asyncio.sleep(delay)
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
|
|
@ -433,7 +430,7 @@ class PryScraper:
|
|||
async def _fetch_via_flaresolverr(self, url: str, timeout: int) -> str:
|
||||
payload = {"cmd": "request.get", "url": url, "maxTimeout": timeout * 1000}
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(timeout + 10)) as client:
|
||||
resp = await client.post(FLARESOLVERR_URL, json=payload)
|
||||
resp = await client.post(settings.flaresolverr_url, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
solution = data.get("solution", {})
|
||||
|
|
|
|||
326
settings.py
326
settings.py
|
|
@ -1,14 +1,60 @@
|
|||
"""Pry — centralized settings via Pydantic BaseSettings.
|
||||
Every env var lives here. Import `settings` singleton everywhere else."""
|
||||
|
||||
Every environment variable lives here. Import the `settings` singleton
|
||||
everywhere else.
|
||||
|
||||
Legacy env vars without the `PRY_` prefix (e.g. `PROXY_URL`, `TOR_ENABLED`)
|
||||
are still accepted during a deprecation window.
|
||||
|
||||
The settings object also loads and persists runtime overrides from a JSON
|
||||
config file (default `/app/config.json`).
|
||||
"""
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import AliasChoices, Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
DEFAULT_CONFIG_FILE = Path("/app/config.json")
|
||||
|
||||
# Map legacy nested config keys (from the old mconfig JSON file and API)
|
||||
# to flat attribute names on PrySettings.
|
||||
_NESTED_KEY_MAP: dict[str, str] = {
|
||||
"proxy.enabled": "proxy_enabled",
|
||||
"proxy.type": "proxy_type",
|
||||
"proxy.url": "proxy_url",
|
||||
"proxy.username": "proxy_username",
|
||||
"proxy.password": "proxy_password",
|
||||
"tor.enabled": "tor_enabled",
|
||||
"tor.socks5_host": "tor_socks5_host",
|
||||
"tor.socks5_port": "tor_socks5_port",
|
||||
"tor.control_port": "tor_control_port",
|
||||
"rotation.user_agent": "rotation_user_agent",
|
||||
"rotation.ip": "ip_rotation",
|
||||
"rotation.timing": "rotation_timing",
|
||||
"rotation.delay_range": "rotation_delay_range",
|
||||
"stealth.webdriver_override": "webdriver_override",
|
||||
"stealth.canvas_noise": "canvas_noise",
|
||||
"stealth.webrtc_disable": "webrtc_disable",
|
||||
"stealth.geolocation_spoof": "geolocation_spoof",
|
||||
"retry.max_attempts": "max_retries",
|
||||
"retry.min_quality": "min_quality",
|
||||
"retry.backoff": "retry_backoff",
|
||||
"output.default_format": "default_format",
|
||||
"output.max_chars": "max_chars",
|
||||
"output.include_links": "include_links",
|
||||
"rate_limit.rpm": "rate_limit_rpm",
|
||||
}
|
||||
|
||||
|
||||
class PrySettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
|
|
@ -17,45 +63,261 @@ class PrySettings(BaseSettings):
|
|||
env_file_encoding="utf-8",
|
||||
case_sensitive=False,
|
||||
extra="ignore",
|
||||
populate_by_name=True,
|
||||
)
|
||||
|
||||
# Core
|
||||
url: str = "http://localhost:8002"
|
||||
app_name: str = "Pry"
|
||||
app_version: str = "3.0.0"
|
||||
environment: Literal["dev", "staging", "prod"] = "dev"
|
||||
log_level: str = "INFO"
|
||||
host: str = "0.0.0.0" # nosec B104
|
||||
port: int = 8002
|
||||
|
||||
# Ollama LLM endpoint
|
||||
ollama_url: str = "http://100.104.130.92:11434"
|
||||
|
||||
# FlareSolverr Cloudflare bypass endpoint
|
||||
flaresolverr_url: str = "http://flaresolverr:8191/v1"
|
||||
|
||||
# Webhook secret for job callbacks
|
||||
webhook_secret: str = "pry-webhook-secret"
|
||||
|
||||
# API key for endpoint authentication (empty = disabled)
|
||||
url: str = "http://localhost:8002"
|
||||
api_key: str = ""
|
||||
|
||||
# OpenRouter API key (optional, for vision models)
|
||||
openrouter_api_key: str = ""
|
||||
|
||||
# Proxy / Tor
|
||||
proxy_url: str = ""
|
||||
proxy_type: str = "http"
|
||||
proxy_username: str = ""
|
||||
proxy_password: str = ""
|
||||
tor_enabled: bool = False
|
||||
tor_socks5_host: str = "tor"
|
||||
tor_socks5_port: int = 9050
|
||||
|
||||
# Rotation
|
||||
ip_rotation: str = ""
|
||||
max_retries: int = 0
|
||||
min_quality: int = 0
|
||||
rate_limit_rpm: int = 0
|
||||
|
||||
# Redis (optional, for job queue persistence)
|
||||
# Database / cache
|
||||
database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry"
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
|
||||
# LLM providers
|
||||
ollama_url: str = "http://100.104.130.92:11434"
|
||||
openrouter_api_key: str = ""
|
||||
openai_api_key: str = ""
|
||||
anthropic_api_key: str = ""
|
||||
cohere_api_key: str = ""
|
||||
|
||||
# Services
|
||||
flaresolverr_url: str = "http://flaresolverr:8191/v1"
|
||||
webhook_secret: str = "pry-webhook-secret"
|
||||
|
||||
# Proxy (legacy env aliases for migration)
|
||||
proxy_url: str = Field(
|
||||
default="",
|
||||
validation_alias=AliasChoices("PRY_PROXY_URL", "PROXY_URL"),
|
||||
)
|
||||
proxy_type: str = Field(
|
||||
default="http",
|
||||
validation_alias=AliasChoices("PRY_PROXY_TYPE", "PROXY_TYPE"),
|
||||
)
|
||||
proxy_username: str = Field(
|
||||
default="",
|
||||
validation_alias=AliasChoices("PRY_PROXY_USERNAME", "PROXY_USERNAME"),
|
||||
)
|
||||
proxy_password: str = Field(
|
||||
default="",
|
||||
validation_alias=AliasChoices("PRY_PROXY_PASSWORD", "PROXY_PASSWORD"),
|
||||
)
|
||||
proxy_enabled: bool = Field(
|
||||
default=False,
|
||||
validation_alias=AliasChoices("PRY_PROXY_ENABLED", "PROXY_ENABLED"),
|
||||
)
|
||||
|
||||
# Tor (legacy env aliases)
|
||||
tor_enabled: bool = Field(
|
||||
default=False,
|
||||
validation_alias=AliasChoices("PRY_TOR_ENABLED", "TOR_ENABLED"),
|
||||
)
|
||||
tor_socks5_host: str = Field(
|
||||
default="tor",
|
||||
validation_alias=AliasChoices("PRY_TOR_SOCKS5_HOST", "TOR_SOCKS5_HOST"),
|
||||
)
|
||||
tor_socks5_port: int = Field(
|
||||
default=9050,
|
||||
validation_alias=AliasChoices("PRY_TOR_SOCKS5_PORT", "TOR_SOCKS5_PORT"),
|
||||
)
|
||||
tor_control_port: int = 9051
|
||||
|
||||
# Rotation / retry (legacy env aliases)
|
||||
rotation_user_agent: str = "rotate"
|
||||
ip_rotation: str = Field(
|
||||
default="off",
|
||||
validation_alias=AliasChoices("PRY_IP_ROTATION", "IP_ROTATION"),
|
||||
)
|
||||
rotation_timing: str = "random"
|
||||
rotation_delay_range: tuple[float, float] = (0.5, 3.0)
|
||||
max_retries: int = Field(
|
||||
default=3,
|
||||
validation_alias=AliasChoices("PRY_MAX_RETRIES", "MAX_RETRIES"),
|
||||
)
|
||||
min_quality: int = Field(
|
||||
default=20,
|
||||
validation_alias=AliasChoices("PRY_MIN_QUALITY", "MIN_QUALITY"),
|
||||
)
|
||||
retry_backoff: str = "exponential"
|
||||
rate_limit_rpm: int = Field(
|
||||
default=120,
|
||||
validation_alias=AliasChoices("PRY_RATE_LIMIT_RPM", "RATE_LIMIT_RPM"),
|
||||
)
|
||||
|
||||
# Stealth
|
||||
stealth_enabled: bool = True
|
||||
random_user_agent: bool = True
|
||||
webdriver_override: bool = True
|
||||
canvas_noise: bool = True
|
||||
webrtc_disable: bool = True
|
||||
geolocation_spoof: bool = True
|
||||
min_delay_ms: int = 500
|
||||
max_delay_ms: int = 3000
|
||||
|
||||
# Output
|
||||
default_format: str = "markdown"
|
||||
max_chars: int = 100000
|
||||
include_links: bool = True
|
||||
|
||||
# Storage
|
||||
screenshot_dir: Path = Path("/tmp/pry-screenshots") # nosec B108
|
||||
cache_ttl_seconds: int = 3600
|
||||
config_file: Path = Field(
|
||||
default=DEFAULT_CONFIG_FILE,
|
||||
validation_alias=AliasChoices("PRY_CONFIG_FILE", "CONFIG_FILE"),
|
||||
)
|
||||
|
||||
# x402 / MCP
|
||||
x402_enabled: bool = False
|
||||
x402_pay_to: str = ""
|
||||
mcp_enabled: bool = True
|
||||
mcp_tools_count: int = 8
|
||||
|
||||
# Rate limits (legacy Settings class)
|
||||
rate_limit_per_domain: int = 10
|
||||
rate_limit_per_ip: int = 60
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
"""Load runtime overrides from the JSON config file after env parsing."""
|
||||
self._load_config_file()
|
||||
|
||||
def _load_config_file(self) -> None:
|
||||
path = self.config_file
|
||||
if not path.exists():
|
||||
return
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
self._apply_nested_updates(data)
|
||||
|
||||
def _apply_nested_updates(self, data: dict[str, Any]) -> None:
|
||||
"""Flatten nested config updates and set attributes on self."""
|
||||
flat: dict[str, Any] = {}
|
||||
|
||||
def _flatten(obj: Any, prefix: str = "") -> None:
|
||||
if isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
new_prefix = f"{prefix}.{key}" if prefix else key
|
||||
_flatten(value, new_prefix)
|
||||
else:
|
||||
flat[prefix] = obj
|
||||
|
||||
_flatten(data)
|
||||
for nested_key, value in flat.items():
|
||||
attr = _NESTED_KEY_MAP.get(nested_key, nested_key)
|
||||
if hasattr(self, attr):
|
||||
if value is None:
|
||||
continue
|
||||
current = getattr(self, attr)
|
||||
expected_type = type(current)
|
||||
try:
|
||||
if expected_type is tuple and isinstance(value, list):
|
||||
setattr(self, attr, tuple(value))
|
||||
else:
|
||||
setattr(self, attr, expected_type(value))
|
||||
except (TypeError, ValueError):
|
||||
setattr(self, attr, value)
|
||||
|
||||
@property
|
||||
def resolved_proxy_chain(self) -> list[str]:
|
||||
"""Return proxy chain: Tor -> user proxy -> direct."""
|
||||
proxies: list[str] = []
|
||||
if self.tor_enabled:
|
||||
proxies.append(f"socks5://{self.tor_socks5_host}:{self.tor_socks5_port}")
|
||||
if self.proxy_enabled and self.proxy_url:
|
||||
if self.proxy_username:
|
||||
netloc = self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url
|
||||
proxies.append(
|
||||
f"{self.proxy_type}://{self.proxy_username}:{self.proxy_password}@{netloc}"
|
||||
)
|
||||
else:
|
||||
proxies.append(self.proxy_url)
|
||||
return proxies
|
||||
|
||||
@property
|
||||
def resolved_proxy_url(self) -> str | None:
|
||||
"""Return the first proxy in the chain, or None for direct connection."""
|
||||
chain = self.resolved_proxy_chain
|
||||
return chain[0] if chain else None
|
||||
|
||||
def to_api_dict(self) -> dict[str, Any]:
|
||||
"""Return the nested config representation used by the /v1/config API."""
|
||||
return {
|
||||
"proxy": {
|
||||
"enabled": self.proxy_enabled,
|
||||
"type": self.proxy_type,
|
||||
"url": self.proxy_url,
|
||||
"username": self.proxy_username,
|
||||
"password": self.proxy_password,
|
||||
},
|
||||
"tor": {
|
||||
"enabled": self.tor_enabled,
|
||||
"socks5_host": self.tor_socks5_host,
|
||||
"socks5_port": self.tor_socks5_port,
|
||||
"control_port": self.tor_control_port,
|
||||
},
|
||||
"rotation": {
|
||||
"user_agent": self.rotation_user_agent,
|
||||
"ip": self.ip_rotation,
|
||||
"timing": self.rotation_timing,
|
||||
"delay_range": list(self.rotation_delay_range),
|
||||
},
|
||||
"stealth": {
|
||||
"webdriver_override": self.webdriver_override,
|
||||
"canvas_noise": self.canvas_noise,
|
||||
"webrtc_disable": self.webrtc_disable,
|
||||
"geolocation_spoof": self.geolocation_spoof,
|
||||
},
|
||||
"retry": {
|
||||
"max_attempts": self.max_retries,
|
||||
"min_quality": self.min_quality,
|
||||
"backoff": self.retry_backoff,
|
||||
},
|
||||
"output": {
|
||||
"default_format": self.default_format,
|
||||
"max_chars": self.max_chars,
|
||||
"include_links": self.include_links,
|
||||
},
|
||||
"rate_limit": {
|
||||
"rpm": self.rate_limit_rpm,
|
||||
},
|
||||
}
|
||||
|
||||
def update(self, updates: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Apply nested runtime config updates and persist to disk."""
|
||||
self._apply_nested_updates(updates)
|
||||
self._persist()
|
||||
return {"status": "ok", "config": self.to_api_dict()}
|
||||
|
||||
def _persist(self) -> None:
|
||||
"""Write the current config to the configured JSON file."""
|
||||
path = self.config_file
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(self.to_api_dict(), f, indent=2)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def enable_tor(self) -> dict[str, Any]:
|
||||
"""Convenience helper used by the /v1/config/profile/tor endpoint."""
|
||||
return self.update(
|
||||
{
|
||||
"tor": {"enabled": True},
|
||||
"proxy": {"enabled": True, "url": "socks5://tor:9050"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# Public singleton. Modules should import this directly.
|
||||
settings = PrySettings()
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
"""Tests for Pry config."""
|
||||
|
||||
from mconfig import PryConfig
|
||||
|
||||
|
||||
def test_config_get_default() -> None:
|
||||
c = PryConfig()
|
||||
assert c.get("nonexistent.key", "fallback") == "fallback"
|
||||
|
||||
|
||||
def test_config_get_retry_exists() -> None:
|
||||
c = PryConfig()
|
||||
val = c.get("retry.max_attempts")
|
||||
assert isinstance(val, int)
|
||||
assert val > 0
|
||||
|
||||
|
||||
def test_config_get_rate_limit_exists() -> None:
|
||||
c = PryConfig()
|
||||
val = c.get("rate_limit.rpm")
|
||||
assert isinstance(val, int)
|
||||
assert val > 0
|
||||
|
||||
|
||||
def test_config_to_dict_excludes_private() -> None:
|
||||
c = PryConfig()
|
||||
d = c.to_dict()
|
||||
assert "_proxy_chain" not in d
|
||||
assert "_proxy_url" not in d
|
||||
|
||||
|
||||
def test_config_update_returns_ok() -> None:
|
||||
c = PryConfig()
|
||||
result = c.update({"retry": {"max_attempts": 5}})
|
||||
assert result["status"] == "ok"
|
||||
assert c.get("retry.max_attempts") == 5
|
||||
|
||||
|
||||
def test_config_get_proxy_url_string_or_none() -> None:
|
||||
c = PryConfig()
|
||||
url = c.get_proxy_url()
|
||||
assert url is None or isinstance(url, str)
|
||||
93
tests/test_settings.py
Normal file
93
tests/test_settings.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
"""Tests for unified Pry settings."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from settings import _NESTED_KEY_MAP, PrySettings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_settings(tmp_path: Path) -> PrySettings:
|
||||
"""Return a fresh settings instance with a temp config file."""
|
||||
config_file = tmp_path / "config.json"
|
||||
return PrySettings(config_file=config_file)
|
||||
|
||||
|
||||
def test_settings_get_default(isolated_settings: PrySettings) -> None:
|
||||
assert isolated_settings.max_retries == 3
|
||||
assert isolated_settings.rate_limit_rpm == 120
|
||||
|
||||
|
||||
def test_settings_proxy_resolution(isolated_settings: PrySettings) -> None:
|
||||
assert isolated_settings.resolved_proxy_url is None
|
||||
isolated_settings.proxy_enabled = True
|
||||
isolated_settings.proxy_url = "http://proxy.example:8080"
|
||||
assert isolated_settings.resolved_proxy_url == "http://proxy.example:8080"
|
||||
|
||||
|
||||
def test_settings_tor_chain(isolated_settings: PrySettings) -> None:
|
||||
isolated_settings.tor_enabled = True
|
||||
isolated_settings.proxy_enabled = True
|
||||
isolated_settings.proxy_url = "http://proxy.example:8080"
|
||||
chain = isolated_settings.resolved_proxy_chain
|
||||
assert len(chain) == 2
|
||||
assert chain[0].startswith("socks5://")
|
||||
|
||||
|
||||
def test_settings_to_api_dict_excludes_private(isolated_settings: PrySettings) -> None:
|
||||
data = isolated_settings.to_api_dict()
|
||||
assert "_proxy_chain" not in data
|
||||
assert "proxy" in data
|
||||
assert isinstance(data["proxy"]["enabled"], bool)
|
||||
|
||||
|
||||
def test_settings_update_and_persist(isolated_settings: PrySettings) -> None:
|
||||
result = isolated_settings.update({"retry": {"max_attempts": 5}})
|
||||
assert result["status"] == "ok"
|
||||
assert isolated_settings.max_retries == 5
|
||||
assert isolated_settings.config_file.exists()
|
||||
saved = json.loads(isolated_settings.config_file.read_text(encoding="utf-8"))
|
||||
assert saved["retry"]["max_attempts"] == 5
|
||||
|
||||
|
||||
def test_settings_load_config_file(tmp_path: Path) -> None:
|
||||
config_file = tmp_path / "config.json"
|
||||
config_file.write_text(
|
||||
json.dumps({"retry": {"max_attempts": 7}, "tor": {"enabled": True}}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
s = PrySettings(config_file=config_file)
|
||||
assert s.max_retries == 7
|
||||
assert s.tor_enabled is True
|
||||
|
||||
|
||||
def test_settings_legacy_env_vars(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setenv("PROXY_URL", "http://legacy-proxy:8080")
|
||||
monkeypatch.setenv("TOR_ENABLED", "true")
|
||||
monkeypatch.setenv("MAX_RETRIES", "9")
|
||||
config_file = tmp_path / "config.json"
|
||||
s = PrySettings(config_file=config_file)
|
||||
assert s.proxy_url == "http://legacy-proxy:8080"
|
||||
assert s.tor_enabled is True
|
||||
assert s.max_retries == 9
|
||||
|
||||
|
||||
def test_settings_enable_tor(isolated_settings: PrySettings) -> None:
|
||||
result = isolated_settings.enable_tor()
|
||||
assert result["status"] == "ok"
|
||||
assert isolated_settings.tor_enabled is True
|
||||
assert isolated_settings.proxy_enabled is True
|
||||
assert isolated_settings.proxy_url == "socks5://tor:9050"
|
||||
|
||||
|
||||
def test_nested_key_map_complete() -> None:
|
||||
"""Ensure every API-facing nested key maps to a settings attribute."""
|
||||
for attr in _NESTED_KEY_MAP.values():
|
||||
assert attr in PrySettings.model_fields, f"PrySettings missing {attr}"
|
||||
Loading…
Add table
Add a link
Reference in a new issue