refactor(settings): unify config.py, mconfig.py, and settings.py into single PrySettings
- Merge all configuration into settings.py with PRY_* env var prefix. - Add legacy env aliases (PROXY_URL, TOR_ENABLED, MAX_RETRIES, etc.) for migration. - Load and persist runtime overrides from JSON config file (default /app/config.json). - Update scraper.py, deps.py, routers/config.py to use unified settings. - Add resolved_proxy_chain and resolved_proxy_url properties. - Replace tests/test_mconfig.py with tests/test_settings.py (9 tests). - Update .env.example with new PRY_* options. - All 500 tests pass; ruff clean.
This commit is contained in:
parent
ecb05bbf49
commit
1f9e71d294
9 changed files with 424 additions and 316 deletions
23
.env.example
23
.env.example
|
|
@ -69,3 +69,26 @@ PRY_FLARESOLVERR_URL=http://flaresolverr:8191/v1
|
||||||
|
|
||||||
# ── Redis ──
|
# ── Redis ──
|
||||||
# PRY_REDIS_URL=redis://localhost:6379/0
|
# PRY_REDIS_URL=redis://localhost:6379/0
|
||||||
|
|
||||||
|
# ── Database ──
|
||||||
|
# PRY_DATABASE_URL=postgresql+asyncpg://pry:pry@localhost/pry
|
||||||
|
|
||||||
|
# ── Stealth ──
|
||||||
|
# PRY_STEALTH_ENABLED=true
|
||||||
|
# PRY_RANDOM_USER_AGENT=true
|
||||||
|
# PRY_WEBDRIVER_OVERRIDE=true
|
||||||
|
# PRY_CANVAS_NOISE=true
|
||||||
|
# PRY_WEBRTC_DISABLE=true
|
||||||
|
# PRY_GEOLOCATION_SPOOF=true
|
||||||
|
# PRY_MIN_DELAY_MS=500
|
||||||
|
# PRY_MAX_DELAY_MS=3000
|
||||||
|
|
||||||
|
# ── Output ──
|
||||||
|
# PRY_DEFAULT_FORMAT=markdown
|
||||||
|
# PRY_MAX_CHARS=100000
|
||||||
|
# PRY_INCLUDE_LINKS=true
|
||||||
|
|
||||||
|
# ── x402 / MCP ──
|
||||||
|
# PRY_X402_ENABLED=false
|
||||||
|
# PRY_X402_PAY_TO=
|
||||||
|
# PRY_MCP_ENABLED=true
|
||||||
|
|
|
||||||
80
config.py
80
config.py
|
|
@ -1,80 +0,0 @@
|
||||||
"""PryScraper — global configuration.
|
|
||||||
|
|
||||||
Loads settings from environment variables (via Pydantic Settings).
|
|
||||||
All secrets come from gopass, never from .env files committed to git.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
# Copyright (c) 2026 Rug Munch Media LLC
|
|
||||||
#
|
|
||||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
||||||
# Licensed under MIT. See LICENSE.
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
|
||||||
"""PryScraper application settings."""
|
|
||||||
|
|
||||||
model_config = SettingsConfigDict(
|
|
||||||
env_file=".env",
|
|
||||||
env_file_encoding="utf-8",
|
|
||||||
case_sensitive=False,
|
|
||||||
extra="ignore",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Core
|
|
||||||
app_name: str = "PryScraper"
|
|
||||||
app_version: str = "3.0.0"
|
|
||||||
environment: Literal["dev", "staging", "prod"] = "dev"
|
|
||||||
log_level: str = "INFO"
|
|
||||||
port: int = 8002
|
|
||||||
|
|
||||||
# Database
|
|
||||||
database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry"
|
|
||||||
|
|
||||||
# Cache
|
|
||||||
redis_url: str = "redis://localhost:6379/0"
|
|
||||||
|
|
||||||
# LLM Providers
|
|
||||||
openai_api_key: str = ""
|
|
||||||
anthropic_api_key: str = ""
|
|
||||||
cohere_api_key: str = ""
|
|
||||||
ollama_url: str = "http://localhost:11434"
|
|
||||||
|
|
||||||
# Stealth
|
|
||||||
stealth_enabled: bool = True
|
|
||||||
random_user_agent: bool = True
|
|
||||||
min_delay_ms: int = 500
|
|
||||||
max_delay_ms: int = 3000
|
|
||||||
|
|
||||||
# Rate limiting
|
|
||||||
rate_limit_per_domain: int = 10 # requests per second
|
|
||||||
rate_limit_per_ip: int = 60
|
|
||||||
|
|
||||||
# x402 Payment
|
|
||||||
x402_enabled: bool = False
|
|
||||||
x402_pay_to: str = ""
|
|
||||||
|
|
||||||
# MCP
|
|
||||||
mcp_enabled: bool = True
|
|
||||||
mcp_tools_count: int = 8
|
|
||||||
|
|
||||||
# Storage
|
|
||||||
screenshot_dir: Path = Path("/tmp/pry-screenshots") # nosec B108
|
|
||||||
cache_ttl_seconds: int = 3600
|
|
||||||
|
|
||||||
|
|
||||||
_settings: Settings | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def get_settings() -> Settings:
|
|
||||||
"""Get cached settings instance."""
|
|
||||||
global _settings
|
|
||||||
if _settings is None:
|
|
||||||
_settings = Settings()
|
|
||||||
return _settings
|
|
||||||
8
deps.py
8
deps.py
|
|
@ -6,8 +6,6 @@ direct access to shared state.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
from deps import scraper, cache, automator, ...
|
from deps import scraper, cache, automator, ...
|
||||||
|
|
||||||
Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# SPDX-License-Identifier: MIT
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
@ -19,18 +17,16 @@ from automator import PryAutomator
|
||||||
from cache import ResponseCache
|
from cache import ResponseCache
|
||||||
from extractor import SchemaExtractor
|
from extractor import SchemaExtractor
|
||||||
from jobqueue import JobQueue
|
from jobqueue import JobQueue
|
||||||
from mconfig import PryConfig
|
|
||||||
from parser import DocumentParser
|
from parser import DocumentParser
|
||||||
from ratelimit import RateLimiter
|
from ratelimit import RateLimiter
|
||||||
from scraper import PryScraper
|
from scraper import PryScraper
|
||||||
|
from settings import settings
|
||||||
config = PryConfig()
|
|
||||||
|
|
||||||
scraper = PryScraper()
|
scraper = PryScraper()
|
||||||
automator = PryAutomator()
|
automator = PryAutomator()
|
||||||
parser = DocumentParser()
|
parser = DocumentParser()
|
||||||
extractor = SchemaExtractor()
|
extractor = SchemaExtractor()
|
||||||
cache = ResponseCache(capacity=1000)
|
cache = ResponseCache(capacity=1000)
|
||||||
ratelimiter = RateLimiter(default_rpm=120, burst=200)
|
ratelimiter = RateLimiter(default_rpm=settings.rate_limit_rpm, burst=200)
|
||||||
queue = JobQueue()
|
queue = JobQueue()
|
||||||
advanced = PryAdvanced(cache=cache)
|
advanced = PryAdvanced(cache=cache)
|
||||||
|
|
|
||||||
133
mconfig.py
133
mconfig.py
|
|
@ -1,133 +0,0 @@
|
||||||
"""Pry Config — runtime configuration for proxy, Tor, VPN, and anti-detection.
|
|
||||||
Users can configure at startup (env vars) or at runtime (API calls)."""
|
|
||||||
|
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
# Copyright (c) 2026 Rug Munch Media LLC
|
|
||||||
#
|
|
||||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
||||||
# Licensed under MIT. See LICENSE.
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
|
|
||||||
CONFIG_FILE = "/app/config.json"
|
|
||||||
DEFAULT_CONFIG = {
|
|
||||||
"proxy": {"enabled": False, "type": "", "url": "", "username": "", "password": ""},
|
|
||||||
"tor": {"enabled": False, "socks5_host": "tor", "socks5_port": 9050, "control_port": 9051},
|
|
||||||
"rotation": {
|
|
||||||
"user_agent": "rotate",
|
|
||||||
"ip": "off",
|
|
||||||
"timing": "random",
|
|
||||||
"delay_range": [0.5, 3.0],
|
|
||||||
},
|
|
||||||
"stealth": {
|
|
||||||
"webdriver_override": True,
|
|
||||||
"canvas_noise": True,
|
|
||||||
"webrtc_disable": True,
|
|
||||||
"geolocation_spoof": True,
|
|
||||||
},
|
|
||||||
"retry": {"max_attempts": 3, "min_quality": 20, "backoff": "exponential"},
|
|
||||||
"output": {"default_format": "markdown", "max_chars": 100000, "include_links": True},
|
|
||||||
"rate_limit": {"rpm": 120, "burst": 200},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class PryConfig:
|
|
||||||
"""Configuration manager — loads from env vars, config file, and API overrides."""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.config = json.loads(json.dumps(DEFAULT_CONFIG)) # Deep copy
|
|
||||||
self._load_env()
|
|
||||||
self._load_file()
|
|
||||||
self._resolve_proxy_chain()
|
|
||||||
|
|
||||||
def _load_env(self):
|
|
||||||
"""Environment variables override defaults."""
|
|
||||||
if os.getenv("TOR_ENABLED", "").lower() in ("1", "true", "yes"):
|
|
||||||
self.config["tor"]["enabled"] = True
|
|
||||||
if os.getenv("TOR_SOCKS5_HOST"):
|
|
||||||
self.config["tor"]["socks5_host"] = os.getenv("TOR_SOCKS5_HOST")
|
|
||||||
if os.getenv("TOR_SOCKS5_PORT"):
|
|
||||||
self.config["tor"]["socks5_port"] = int(os.getenv("TOR_SOCKS5_PORT"))
|
|
||||||
if os.getenv("PROXY_URL"):
|
|
||||||
self.config["proxy"]["enabled"] = True
|
|
||||||
self.config["proxy"]["url"] = os.getenv("PROXY_URL")
|
|
||||||
self.config["proxy"]["type"] = os.getenv("PROXY_TYPE", "http")
|
|
||||||
if os.getenv("PROXY_USERNAME"):
|
|
||||||
self.config["proxy"]["username"] = os.getenv("PROXY_USERNAME")
|
|
||||||
if os.getenv("PROXY_PASSWORD"):
|
|
||||||
self.config["proxy"]["password"] = os.getenv("PROXY_PASSWORD")
|
|
||||||
if os.getenv("IP_ROTATION"):
|
|
||||||
self.config["rotation"]["ip"] = os.getenv("IP_ROTATION")
|
|
||||||
if os.getenv("MAX_RETRIES"):
|
|
||||||
self.config["retry"]["max_attempts"] = int(os.getenv("MAX_RETRIES"))
|
|
||||||
if os.getenv("MIN_QUALITY"):
|
|
||||||
self.config["retry"]["min_quality"] = int(os.getenv("MIN_QUALITY"))
|
|
||||||
if os.getenv("RATE_LIMIT_RPM"):
|
|
||||||
self.config["rate_limit"]["rpm"] = int(os.getenv("RATE_LIMIT_RPM"))
|
|
||||||
|
|
||||||
def _load_file(self):
|
|
||||||
if os.path.exists(CONFIG_FILE):
|
|
||||||
try:
|
|
||||||
with open(CONFIG_FILE) as f:
|
|
||||||
user_config = json.load(f)
|
|
||||||
self._deep_merge(self.config, user_config)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def _deep_merge(self, base: dict, override: dict):
|
|
||||||
for key, value in override.items():
|
|
||||||
if key in base and isinstance(base[key], dict) and isinstance(value, dict):
|
|
||||||
self._deep_merge(base[key], value)
|
|
||||||
else:
|
|
||||||
base[key] = value
|
|
||||||
|
|
||||||
def _resolve_proxy_chain(self):
|
|
||||||
"""Build the proxy chain: Tor -> User Proxy -> Direct"""
|
|
||||||
proxies = []
|
|
||||||
if self.config["tor"]["enabled"]:
|
|
||||||
proxies.append(
|
|
||||||
f"socks5://{self.config['tor']['socks5_host']}:{self.config['tor']['socks5_port']}"
|
|
||||||
)
|
|
||||||
if self.config["proxy"]["enabled"]:
|
|
||||||
url = self.config["proxy"]["url"]
|
|
||||||
if self.config["proxy"]["username"]:
|
|
||||||
netloc = url.split("://")[1] if "://" in url else url
|
|
||||||
proxies.append(
|
|
||||||
f"{self.config['proxy']['type']}://{self.config['proxy']['username']}:{self.config['proxy']['password']}@{netloc}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
proxies.append(url)
|
|
||||||
self.config["_proxy_chain"] = proxies
|
|
||||||
self.config["_proxy_url"] = proxies[0] if proxies else None
|
|
||||||
|
|
||||||
def get_proxy_url(self) -> str | None:
|
|
||||||
return self.config.get("_proxy_url")
|
|
||||||
|
|
||||||
def get_proxy_chain(self) -> list:
|
|
||||||
return self.config.get("_proxy_chain", [])
|
|
||||||
|
|
||||||
def get(self, key: str, default=None):
|
|
||||||
keys = key.split(".")
|
|
||||||
val = self.config
|
|
||||||
for k in keys:
|
|
||||||
if isinstance(val, dict):
|
|
||||||
val = val.get(k)
|
|
||||||
else:
|
|
||||||
return default
|
|
||||||
if val is None:
|
|
||||||
return default
|
|
||||||
return val if val is not None else default
|
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
|
||||||
return {k: v for k, v in self.config.items() if not k.startswith("_")}
|
|
||||||
|
|
||||||
def update(self, updates: dict) -> dict:
|
|
||||||
self._deep_merge(self.config, updates)
|
|
||||||
self._resolve_proxy_chain()
|
|
||||||
try:
|
|
||||||
with open(CONFIG_FILE, "w") as f:
|
|
||||||
json.dump(self.to_dict(), f, indent=2)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
return {"status": "ok", "config": self.to_dict()}
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
"""Pry — Config router (remaining api.py routes).
|
"""Pry — Config router (runtime configuration management).
|
||||||
|
|
||||||
Auto-extracted from api.py during the router-split refactor.
|
Provides endpoints to read and update the unified Pry settings object.
|
||||||
|
Runtime changes are persisted to the configured JSON file.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# SPDX-License-Identifier: MIT
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
@ -13,37 +14,33 @@ from typing import Any
|
||||||
|
|
||||||
from fastapi import APIRouter, Body
|
from fastapi import APIRouter, Body
|
||||||
|
|
||||||
from mconfig import PryConfig
|
from settings import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
router = APIRouter(tags=["Config"])
|
router = APIRouter(tags=["Config"])
|
||||||
|
|
||||||
|
|
||||||
@router.get("/v1/config", tags=["Config"], summary="Get current Pry configuration")
|
@router.get("/v1/config", tags=["Config"], summary="Get current Pry configuration")
|
||||||
async def get_config() -> dict[str, Any]:
|
async def get_config() -> dict[str, Any]:
|
||||||
"""Get current Pry configuration."""
|
"""Get current Pry configuration."""
|
||||||
return {"success": True, "data": config.to_dict()}
|
return {"success": True, "data": settings.to_api_dict()}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime")
|
@router.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime")
|
||||||
async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
||||||
"""Update Pry configuration at runtime."""
|
"""Update Pry configuration at runtime."""
|
||||||
result = config.update(updates)
|
result = settings.update(updates)
|
||||||
return {"success": True, "data": result}
|
return {"success": True, "data": result}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests")
|
@router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests")
|
||||||
async def enable_tor() -> dict[str, Any]:
|
async def enable_tor() -> dict[str, Any]:
|
||||||
"""Enable Tor routing for all requests."""
|
"""Enable Tor routing for all requests."""
|
||||||
result = config.update(
|
result = settings.enable_tor()
|
||||||
{"tor": {"enabled": True}, "proxy": {"enabled": True, "url": "socks5://tor:9050"}}
|
|
||||||
)
|
|
||||||
resp_data = {
|
resp_data = {
|
||||||
"success": True,
|
"success": True,
|
||||||
"data": result,
|
"data": result,
|
||||||
"note": "Run 'docker compose --profile tor up -d' to start Tor container",
|
"note": "Run 'docker compose --profile tor up -d' to start Tor container",
|
||||||
}
|
}
|
||||||
return resp_data
|
return resp_data
|
||||||
|
|
||||||
|
|
||||||
config = PryConfig()
|
|
||||||
|
|
|
||||||
11
scraper.py
11
scraper.py
|
|
@ -19,10 +19,7 @@ import httpx
|
||||||
import trafilatura
|
import trafilatura
|
||||||
from trafilatura.settings import use_config
|
from trafilatura.settings import use_config
|
||||||
|
|
||||||
from mconfig import PryConfig
|
from settings import settings
|
||||||
|
|
||||||
FLARESOLVERR_URL = "http://flaresolverr:8191/v1"
|
|
||||||
config = PryConfig()
|
|
||||||
|
|
||||||
USER_AGENTS = [
|
USER_AGENTS = [
|
||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||||
|
|
@ -415,7 +412,7 @@ class PryScraper:
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def _fetch_direct(self, url: str, headers: dict, timeout: int) -> str:
|
async def _fetch_direct(self, url: str, headers: dict, timeout: int) -> str:
|
||||||
proxy_url = config.get_proxy_url()
|
proxy_url = settings.resolved_proxy_url
|
||||||
async with httpx.AsyncClient(
|
async with httpx.AsyncClient(
|
||||||
timeout=httpx.Timeout(timeout),
|
timeout=httpx.Timeout(timeout),
|
||||||
follow_redirects=True,
|
follow_redirects=True,
|
||||||
|
|
@ -424,7 +421,7 @@ class PryScraper:
|
||||||
cookies=httpx.Cookies(),
|
cookies=httpx.Cookies(),
|
||||||
proxy=proxy_url,
|
proxy=proxy_url,
|
||||||
) as client:
|
) as client:
|
||||||
delay = random.uniform(*config.get("rotation.delay_range", [0.5, 3.0]))
|
delay = random.uniform(*settings.rotation_delay_range)
|
||||||
await asyncio.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
resp = await client.get(url)
|
resp = await client.get(url)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
|
|
@ -433,7 +430,7 @@ class PryScraper:
|
||||||
async def _fetch_via_flaresolverr(self, url: str, timeout: int) -> str:
|
async def _fetch_via_flaresolverr(self, url: str, timeout: int) -> str:
|
||||||
payload = {"cmd": "request.get", "url": url, "maxTimeout": timeout * 1000}
|
payload = {"cmd": "request.get", "url": url, "maxTimeout": timeout * 1000}
|
||||||
async with httpx.AsyncClient(timeout=httpx.Timeout(timeout + 10)) as client:
|
async with httpx.AsyncClient(timeout=httpx.Timeout(timeout + 10)) as client:
|
||||||
resp = await client.post(FLARESOLVERR_URL, json=payload)
|
resp = await client.post(settings.flaresolverr_url, json=payload)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
solution = data.get("solution", {})
|
solution = data.get("solution", {})
|
||||||
|
|
|
||||||
326
settings.py
326
settings.py
|
|
@ -1,14 +1,60 @@
|
||||||
"""Pry — centralized settings via Pydantic BaseSettings.
|
"""Pry — centralized settings via Pydantic BaseSettings.
|
||||||
Every env var lives here. Import `settings` singleton everywhere else."""
|
|
||||||
|
Every environment variable lives here. Import the `settings` singleton
|
||||||
|
everywhere else.
|
||||||
|
|
||||||
|
Legacy env vars without the `PRY_` prefix (e.g. `PROXY_URL`, `TOR_ENABLED`)
|
||||||
|
are still accepted during a deprecation window.
|
||||||
|
|
||||||
|
The settings object also loads and persists runtime overrides from a JSON
|
||||||
|
config file (default `/app/config.json`).
|
||||||
|
"""
|
||||||
|
|
||||||
# SPDX-License-Identifier: MIT
|
# SPDX-License-Identifier: MIT
|
||||||
# Copyright (c) 2026 Rug Munch Media LLC
|
# Copyright (c) 2026 Rug Munch Media LLC
|
||||||
#
|
#
|
||||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||||
# Licensed under MIT. See LICENSE.
|
# Licensed under MIT. See LICENSE.
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from pydantic import AliasChoices, Field
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
|
DEFAULT_CONFIG_FILE = Path("/app/config.json")
|
||||||
|
|
||||||
|
# Map legacy nested config keys (from the old mconfig JSON file and API)
|
||||||
|
# to flat attribute names on PrySettings.
|
||||||
|
_NESTED_KEY_MAP: dict[str, str] = {
|
||||||
|
"proxy.enabled": "proxy_enabled",
|
||||||
|
"proxy.type": "proxy_type",
|
||||||
|
"proxy.url": "proxy_url",
|
||||||
|
"proxy.username": "proxy_username",
|
||||||
|
"proxy.password": "proxy_password",
|
||||||
|
"tor.enabled": "tor_enabled",
|
||||||
|
"tor.socks5_host": "tor_socks5_host",
|
||||||
|
"tor.socks5_port": "tor_socks5_port",
|
||||||
|
"tor.control_port": "tor_control_port",
|
||||||
|
"rotation.user_agent": "rotation_user_agent",
|
||||||
|
"rotation.ip": "ip_rotation",
|
||||||
|
"rotation.timing": "rotation_timing",
|
||||||
|
"rotation.delay_range": "rotation_delay_range",
|
||||||
|
"stealth.webdriver_override": "webdriver_override",
|
||||||
|
"stealth.canvas_noise": "canvas_noise",
|
||||||
|
"stealth.webrtc_disable": "webrtc_disable",
|
||||||
|
"stealth.geolocation_spoof": "geolocation_spoof",
|
||||||
|
"retry.max_attempts": "max_retries",
|
||||||
|
"retry.min_quality": "min_quality",
|
||||||
|
"retry.backoff": "retry_backoff",
|
||||||
|
"output.default_format": "default_format",
|
||||||
|
"output.max_chars": "max_chars",
|
||||||
|
"output.include_links": "include_links",
|
||||||
|
"rate_limit.rpm": "rate_limit_rpm",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class PrySettings(BaseSettings):
|
class PrySettings(BaseSettings):
|
||||||
model_config = SettingsConfigDict(
|
model_config = SettingsConfigDict(
|
||||||
|
|
@ -17,45 +63,261 @@ class PrySettings(BaseSettings):
|
||||||
env_file_encoding="utf-8",
|
env_file_encoding="utf-8",
|
||||||
case_sensitive=False,
|
case_sensitive=False,
|
||||||
extra="ignore",
|
extra="ignore",
|
||||||
|
populate_by_name=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Core
|
# Core
|
||||||
url: str = "http://localhost:8002"
|
app_name: str = "Pry"
|
||||||
|
app_version: str = "3.0.0"
|
||||||
|
environment: Literal["dev", "staging", "prod"] = "dev"
|
||||||
|
log_level: str = "INFO"
|
||||||
host: str = "0.0.0.0" # nosec B104
|
host: str = "0.0.0.0" # nosec B104
|
||||||
port: int = 8002
|
port: int = 8002
|
||||||
|
url: str = "http://localhost:8002"
|
||||||
# Ollama LLM endpoint
|
|
||||||
ollama_url: str = "http://100.104.130.92:11434"
|
|
||||||
|
|
||||||
# FlareSolverr Cloudflare bypass endpoint
|
|
||||||
flaresolverr_url: str = "http://flaresolverr:8191/v1"
|
|
||||||
|
|
||||||
# Webhook secret for job callbacks
|
|
||||||
webhook_secret: str = "pry-webhook-secret"
|
|
||||||
|
|
||||||
# API key for endpoint authentication (empty = disabled)
|
|
||||||
api_key: str = ""
|
api_key: str = ""
|
||||||
|
|
||||||
# OpenRouter API key (optional, for vision models)
|
# Database / cache
|
||||||
openrouter_api_key: str = ""
|
database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry"
|
||||||
|
|
||||||
# Proxy / Tor
|
|
||||||
proxy_url: str = ""
|
|
||||||
proxy_type: str = "http"
|
|
||||||
proxy_username: str = ""
|
|
||||||
proxy_password: str = ""
|
|
||||||
tor_enabled: bool = False
|
|
||||||
tor_socks5_host: str = "tor"
|
|
||||||
tor_socks5_port: int = 9050
|
|
||||||
|
|
||||||
# Rotation
|
|
||||||
ip_rotation: str = ""
|
|
||||||
max_retries: int = 0
|
|
||||||
min_quality: int = 0
|
|
||||||
rate_limit_rpm: int = 0
|
|
||||||
|
|
||||||
# Redis (optional, for job queue persistence)
|
|
||||||
redis_url: str = "redis://localhost:6379/0"
|
redis_url: str = "redis://localhost:6379/0"
|
||||||
|
|
||||||
|
# LLM providers
|
||||||
|
ollama_url: str = "http://100.104.130.92:11434"
|
||||||
|
openrouter_api_key: str = ""
|
||||||
|
openai_api_key: str = ""
|
||||||
|
anthropic_api_key: str = ""
|
||||||
|
cohere_api_key: str = ""
|
||||||
|
|
||||||
|
# Services
|
||||||
|
flaresolverr_url: str = "http://flaresolverr:8191/v1"
|
||||||
|
webhook_secret: str = "pry-webhook-secret"
|
||||||
|
|
||||||
|
# Proxy (legacy env aliases for migration)
|
||||||
|
proxy_url: str = Field(
|
||||||
|
default="",
|
||||||
|
validation_alias=AliasChoices("PRY_PROXY_URL", "PROXY_URL"),
|
||||||
|
)
|
||||||
|
proxy_type: str = Field(
|
||||||
|
default="http",
|
||||||
|
validation_alias=AliasChoices("PRY_PROXY_TYPE", "PROXY_TYPE"),
|
||||||
|
)
|
||||||
|
proxy_username: str = Field(
|
||||||
|
default="",
|
||||||
|
validation_alias=AliasChoices("PRY_PROXY_USERNAME", "PROXY_USERNAME"),
|
||||||
|
)
|
||||||
|
proxy_password: str = Field(
|
||||||
|
default="",
|
||||||
|
validation_alias=AliasChoices("PRY_PROXY_PASSWORD", "PROXY_PASSWORD"),
|
||||||
|
)
|
||||||
|
proxy_enabled: bool = Field(
|
||||||
|
default=False,
|
||||||
|
validation_alias=AliasChoices("PRY_PROXY_ENABLED", "PROXY_ENABLED"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Tor (legacy env aliases)
|
||||||
|
tor_enabled: bool = Field(
|
||||||
|
default=False,
|
||||||
|
validation_alias=AliasChoices("PRY_TOR_ENABLED", "TOR_ENABLED"),
|
||||||
|
)
|
||||||
|
tor_socks5_host: str = Field(
|
||||||
|
default="tor",
|
||||||
|
validation_alias=AliasChoices("PRY_TOR_SOCKS5_HOST", "TOR_SOCKS5_HOST"),
|
||||||
|
)
|
||||||
|
tor_socks5_port: int = Field(
|
||||||
|
default=9050,
|
||||||
|
validation_alias=AliasChoices("PRY_TOR_SOCKS5_PORT", "TOR_SOCKS5_PORT"),
|
||||||
|
)
|
||||||
|
tor_control_port: int = 9051
|
||||||
|
|
||||||
|
# Rotation / retry (legacy env aliases)
|
||||||
|
rotation_user_agent: str = "rotate"
|
||||||
|
ip_rotation: str = Field(
|
||||||
|
default="off",
|
||||||
|
validation_alias=AliasChoices("PRY_IP_ROTATION", "IP_ROTATION"),
|
||||||
|
)
|
||||||
|
rotation_timing: str = "random"
|
||||||
|
rotation_delay_range: tuple[float, float] = (0.5, 3.0)
|
||||||
|
max_retries: int = Field(
|
||||||
|
default=3,
|
||||||
|
validation_alias=AliasChoices("PRY_MAX_RETRIES", "MAX_RETRIES"),
|
||||||
|
)
|
||||||
|
min_quality: int = Field(
|
||||||
|
default=20,
|
||||||
|
validation_alias=AliasChoices("PRY_MIN_QUALITY", "MIN_QUALITY"),
|
||||||
|
)
|
||||||
|
retry_backoff: str = "exponential"
|
||||||
|
rate_limit_rpm: int = Field(
|
||||||
|
default=120,
|
||||||
|
validation_alias=AliasChoices("PRY_RATE_LIMIT_RPM", "RATE_LIMIT_RPM"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Stealth
|
||||||
|
stealth_enabled: bool = True
|
||||||
|
random_user_agent: bool = True
|
||||||
|
webdriver_override: bool = True
|
||||||
|
canvas_noise: bool = True
|
||||||
|
webrtc_disable: bool = True
|
||||||
|
geolocation_spoof: bool = True
|
||||||
|
min_delay_ms: int = 500
|
||||||
|
max_delay_ms: int = 3000
|
||||||
|
|
||||||
|
# Output
|
||||||
|
default_format: str = "markdown"
|
||||||
|
max_chars: int = 100000
|
||||||
|
include_links: bool = True
|
||||||
|
|
||||||
|
# Storage
|
||||||
|
screenshot_dir: Path = Path("/tmp/pry-screenshots") # nosec B108
|
||||||
|
cache_ttl_seconds: int = 3600
|
||||||
|
config_file: Path = Field(
|
||||||
|
default=DEFAULT_CONFIG_FILE,
|
||||||
|
validation_alias=AliasChoices("PRY_CONFIG_FILE", "CONFIG_FILE"),
|
||||||
|
)
|
||||||
|
|
||||||
|
# x402 / MCP
|
||||||
|
x402_enabled: bool = False
|
||||||
|
x402_pay_to: str = ""
|
||||||
|
mcp_enabled: bool = True
|
||||||
|
mcp_tools_count: int = 8
|
||||||
|
|
||||||
|
# Rate limits (legacy Settings class)
|
||||||
|
rate_limit_per_domain: int = 10
|
||||||
|
rate_limit_per_ip: int = 60
|
||||||
|
|
||||||
|
def model_post_init(self, __context: Any) -> None:
|
||||||
|
"""Load runtime overrides from the JSON config file after env parsing."""
|
||||||
|
self._load_config_file()
|
||||||
|
|
||||||
|
def _load_config_file(self) -> None:
|
||||||
|
path = self.config_file
|
||||||
|
if not path.exists():
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with open(path, encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return
|
||||||
|
self._apply_nested_updates(data)
|
||||||
|
|
||||||
|
def _apply_nested_updates(self, data: dict[str, Any]) -> None:
|
||||||
|
"""Flatten nested config updates and set attributes on self."""
|
||||||
|
flat: dict[str, Any] = {}
|
||||||
|
|
||||||
|
def _flatten(obj: Any, prefix: str = "") -> None:
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
for key, value in obj.items():
|
||||||
|
new_prefix = f"{prefix}.{key}" if prefix else key
|
||||||
|
_flatten(value, new_prefix)
|
||||||
|
else:
|
||||||
|
flat[prefix] = obj
|
||||||
|
|
||||||
|
_flatten(data)
|
||||||
|
for nested_key, value in flat.items():
|
||||||
|
attr = _NESTED_KEY_MAP.get(nested_key, nested_key)
|
||||||
|
if hasattr(self, attr):
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
current = getattr(self, attr)
|
||||||
|
expected_type = type(current)
|
||||||
|
try:
|
||||||
|
if expected_type is tuple and isinstance(value, list):
|
||||||
|
setattr(self, attr, tuple(value))
|
||||||
|
else:
|
||||||
|
setattr(self, attr, expected_type(value))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
setattr(self, attr, value)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def resolved_proxy_chain(self) -> list[str]:
|
||||||
|
"""Return proxy chain: Tor -> user proxy -> direct."""
|
||||||
|
proxies: list[str] = []
|
||||||
|
if self.tor_enabled:
|
||||||
|
proxies.append(f"socks5://{self.tor_socks5_host}:{self.tor_socks5_port}")
|
||||||
|
if self.proxy_enabled and self.proxy_url:
|
||||||
|
if self.proxy_username:
|
||||||
|
netloc = self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url
|
||||||
|
proxies.append(
|
||||||
|
f"{self.proxy_type}://{self.proxy_username}:{self.proxy_password}@{netloc}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
proxies.append(self.proxy_url)
|
||||||
|
return proxies
|
||||||
|
|
||||||
|
@property
|
||||||
|
def resolved_proxy_url(self) -> str | None:
|
||||||
|
"""Return the first proxy in the chain, or None for direct connection."""
|
||||||
|
chain = self.resolved_proxy_chain
|
||||||
|
return chain[0] if chain else None
|
||||||
|
|
||||||
|
def to_api_dict(self) -> dict[str, Any]:
|
||||||
|
"""Return the nested config representation used by the /v1/config API."""
|
||||||
|
return {
|
||||||
|
"proxy": {
|
||||||
|
"enabled": self.proxy_enabled,
|
||||||
|
"type": self.proxy_type,
|
||||||
|
"url": self.proxy_url,
|
||||||
|
"username": self.proxy_username,
|
||||||
|
"password": self.proxy_password,
|
||||||
|
},
|
||||||
|
"tor": {
|
||||||
|
"enabled": self.tor_enabled,
|
||||||
|
"socks5_host": self.tor_socks5_host,
|
||||||
|
"socks5_port": self.tor_socks5_port,
|
||||||
|
"control_port": self.tor_control_port,
|
||||||
|
},
|
||||||
|
"rotation": {
|
||||||
|
"user_agent": self.rotation_user_agent,
|
||||||
|
"ip": self.ip_rotation,
|
||||||
|
"timing": self.rotation_timing,
|
||||||
|
"delay_range": list(self.rotation_delay_range),
|
||||||
|
},
|
||||||
|
"stealth": {
|
||||||
|
"webdriver_override": self.webdriver_override,
|
||||||
|
"canvas_noise": self.canvas_noise,
|
||||||
|
"webrtc_disable": self.webrtc_disable,
|
||||||
|
"geolocation_spoof": self.geolocation_spoof,
|
||||||
|
},
|
||||||
|
"retry": {
|
||||||
|
"max_attempts": self.max_retries,
|
||||||
|
"min_quality": self.min_quality,
|
||||||
|
"backoff": self.retry_backoff,
|
||||||
|
},
|
||||||
|
"output": {
|
||||||
|
"default_format": self.default_format,
|
||||||
|
"max_chars": self.max_chars,
|
||||||
|
"include_links": self.include_links,
|
||||||
|
},
|
||||||
|
"rate_limit": {
|
||||||
|
"rpm": self.rate_limit_rpm,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def update(self, updates: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Apply nested runtime config updates and persist to disk."""
|
||||||
|
self._apply_nested_updates(updates)
|
||||||
|
self._persist()
|
||||||
|
return {"status": "ok", "config": self.to_api_dict()}
|
||||||
|
|
||||||
|
def _persist(self) -> None:
|
||||||
|
"""Write the current config to the configured JSON file."""
|
||||||
|
path = self.config_file
|
||||||
|
try:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(self.to_api_dict(), f, indent=2)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def enable_tor(self) -> dict[str, Any]:
|
||||||
|
"""Convenience helper used by the /v1/config/profile/tor endpoint."""
|
||||||
|
return self.update(
|
||||||
|
{
|
||||||
|
"tor": {"enabled": True},
|
||||||
|
"proxy": {"enabled": True, "url": "socks5://tor:9050"},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Public singleton. Modules should import this directly.
|
||||||
settings = PrySettings()
|
settings = PrySettings()
|
||||||
|
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
# SPDX-License-Identifier: MIT
|
|
||||||
# Copyright (c) 2026 Rug Munch Media LLC
|
|
||||||
#
|
|
||||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
||||||
# Licensed under MIT. See LICENSE.
|
|
||||||
"""Tests for Pry config."""
|
|
||||||
|
|
||||||
from mconfig import PryConfig
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_get_default() -> None:
|
|
||||||
c = PryConfig()
|
|
||||||
assert c.get("nonexistent.key", "fallback") == "fallback"
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_get_retry_exists() -> None:
|
|
||||||
c = PryConfig()
|
|
||||||
val = c.get("retry.max_attempts")
|
|
||||||
assert isinstance(val, int)
|
|
||||||
assert val > 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_get_rate_limit_exists() -> None:
|
|
||||||
c = PryConfig()
|
|
||||||
val = c.get("rate_limit.rpm")
|
|
||||||
assert isinstance(val, int)
|
|
||||||
assert val > 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_to_dict_excludes_private() -> None:
|
|
||||||
c = PryConfig()
|
|
||||||
d = c.to_dict()
|
|
||||||
assert "_proxy_chain" not in d
|
|
||||||
assert "_proxy_url" not in d
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_update_returns_ok() -> None:
|
|
||||||
c = PryConfig()
|
|
||||||
result = c.update({"retry": {"max_attempts": 5}})
|
|
||||||
assert result["status"] == "ok"
|
|
||||||
assert c.get("retry.max_attempts") == 5
|
|
||||||
|
|
||||||
|
|
||||||
def test_config_get_proxy_url_string_or_none() -> None:
|
|
||||||
c = PryConfig()
|
|
||||||
url = c.get_proxy_url()
|
|
||||||
assert url is None or isinstance(url, str)
|
|
||||||
93
tests/test_settings.py
Normal file
93
tests/test_settings.py
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
# Copyright (c) 2026 Rug Munch Media LLC
|
||||||
|
#
|
||||||
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||||
|
# Licensed under MIT. See LICENSE.
|
||||||
|
"""Tests for unified Pry settings."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from settings import _NESTED_KEY_MAP, PrySettings
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def isolated_settings(tmp_path: Path) -> PrySettings:
|
||||||
|
"""Return a fresh settings instance with a temp config file."""
|
||||||
|
config_file = tmp_path / "config.json"
|
||||||
|
return PrySettings(config_file=config_file)
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_get_default(isolated_settings: PrySettings) -> None:
|
||||||
|
assert isolated_settings.max_retries == 3
|
||||||
|
assert isolated_settings.rate_limit_rpm == 120
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_proxy_resolution(isolated_settings: PrySettings) -> None:
|
||||||
|
assert isolated_settings.resolved_proxy_url is None
|
||||||
|
isolated_settings.proxy_enabled = True
|
||||||
|
isolated_settings.proxy_url = "http://proxy.example:8080"
|
||||||
|
assert isolated_settings.resolved_proxy_url == "http://proxy.example:8080"
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_tor_chain(isolated_settings: PrySettings) -> None:
|
||||||
|
isolated_settings.tor_enabled = True
|
||||||
|
isolated_settings.proxy_enabled = True
|
||||||
|
isolated_settings.proxy_url = "http://proxy.example:8080"
|
||||||
|
chain = isolated_settings.resolved_proxy_chain
|
||||||
|
assert len(chain) == 2
|
||||||
|
assert chain[0].startswith("socks5://")
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_to_api_dict_excludes_private(isolated_settings: PrySettings) -> None:
|
||||||
|
data = isolated_settings.to_api_dict()
|
||||||
|
assert "_proxy_chain" not in data
|
||||||
|
assert "proxy" in data
|
||||||
|
assert isinstance(data["proxy"]["enabled"], bool)
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_update_and_persist(isolated_settings: PrySettings) -> None:
|
||||||
|
result = isolated_settings.update({"retry": {"max_attempts": 5}})
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert isolated_settings.max_retries == 5
|
||||||
|
assert isolated_settings.config_file.exists()
|
||||||
|
saved = json.loads(isolated_settings.config_file.read_text(encoding="utf-8"))
|
||||||
|
assert saved["retry"]["max_attempts"] == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_load_config_file(tmp_path: Path) -> None:
|
||||||
|
config_file = tmp_path / "config.json"
|
||||||
|
config_file.write_text(
|
||||||
|
json.dumps({"retry": {"max_attempts": 7}, "tor": {"enabled": True}}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
s = PrySettings(config_file=config_file)
|
||||||
|
assert s.max_retries == 7
|
||||||
|
assert s.tor_enabled is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_legacy_env_vars(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
monkeypatch.setenv("PROXY_URL", "http://legacy-proxy:8080")
|
||||||
|
monkeypatch.setenv("TOR_ENABLED", "true")
|
||||||
|
monkeypatch.setenv("MAX_RETRIES", "9")
|
||||||
|
config_file = tmp_path / "config.json"
|
||||||
|
s = PrySettings(config_file=config_file)
|
||||||
|
assert s.proxy_url == "http://legacy-proxy:8080"
|
||||||
|
assert s.tor_enabled is True
|
||||||
|
assert s.max_retries == 9
|
||||||
|
|
||||||
|
|
||||||
|
def test_settings_enable_tor(isolated_settings: PrySettings) -> None:
|
||||||
|
result = isolated_settings.enable_tor()
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert isolated_settings.tor_enabled is True
|
||||||
|
assert isolated_settings.proxy_enabled is True
|
||||||
|
assert isolated_settings.proxy_url == "socks5://tor:9050"
|
||||||
|
|
||||||
|
|
||||||
|
def test_nested_key_map_complete() -> None:
|
||||||
|
"""Ensure every API-facing nested key maps to a settings attribute."""
|
||||||
|
for attr in _NESTED_KEY_MAP.values():
|
||||||
|
assert attr in PrySettings.model_fields, f"PrySettings missing {attr}"
|
||||||
Loading…
Add table
Add a link
Reference in a new issue