From 1f9e71d294416fde5a09df0971328fb14dc0b1c1 Mon Sep 17 00:00:00 2001 From: cryptorugmunch Date: Fri, 3 Jul 2026 05:04:29 +0200 Subject: [PATCH] refactor(settings): unify config.py, mconfig.py, and settings.py into single PrySettings - Merge all configuration into settings.py with PRY_* env var prefix. - Add legacy env aliases (PROXY_URL, TOR_ENABLED, MAX_RETRIES, etc.) for migration. - Load and persist runtime overrides from JSON config file (default /app/config.json). - Update scraper.py, deps.py, routers/config.py to use unified settings. - Add resolved_proxy_chain and resolved_proxy_url properties. - Replace tests/test_mconfig.py with tests/test_settings.py (9 tests). - Update .env.example with new PRY_* options. - All 500 tests pass; ruff clean. --- .env.example | 23 +++ config.py | 80 ---------- deps.py | 8 +- mconfig.py | 133 ----------------- routers/config.py | 19 +-- scraper.py | 11 +- settings.py | 326 +++++++++++++++++++++++++++++++++++++---- tests/test_mconfig.py | 47 ------ tests/test_settings.py | 93 ++++++++++++ 9 files changed, 424 insertions(+), 316 deletions(-) delete mode 100644 config.py delete mode 100644 mconfig.py delete mode 100644 tests/test_mconfig.py create mode 100644 tests/test_settings.py diff --git a/.env.example b/.env.example index 355d334..8fd3fdc 100644 --- a/.env.example +++ b/.env.example @@ -69,3 +69,26 @@ PRY_FLARESOLVERR_URL=http://flaresolverr:8191/v1 # ── Redis ── # PRY_REDIS_URL=redis://localhost:6379/0 + +# ── Database ── +# PRY_DATABASE_URL=postgresql+asyncpg://pry:pry@localhost/pry + +# ── Stealth ── +# PRY_STEALTH_ENABLED=true +# PRY_RANDOM_USER_AGENT=true +# PRY_WEBDRIVER_OVERRIDE=true +# PRY_CANVAS_NOISE=true +# PRY_WEBRTC_DISABLE=true +# PRY_GEOLOCATION_SPOOF=true +# PRY_MIN_DELAY_MS=500 +# PRY_MAX_DELAY_MS=3000 + +# ── Output ── +# PRY_DEFAULT_FORMAT=markdown +# PRY_MAX_CHARS=100000 +# PRY_INCLUDE_LINKS=true + +# ── x402 / MCP ── +# PRY_X402_ENABLED=false +# PRY_X402_PAY_TO= +# PRY_MCP_ENABLED=true diff --git a/config.py b/config.py deleted file mode 100644 index 236a270..0000000 --- a/config.py +++ /dev/null @@ -1,80 +0,0 @@ -"""PryScraper — global configuration. - -Loads settings from environment variables (via Pydantic Settings). -All secrets come from gopass, never from .env files committed to git. -""" - -# SPDX-License-Identifier: MIT -# Copyright (c) 2026 Rug Munch Media LLC -# -# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper -# Licensed under MIT. See LICENSE. -from __future__ import annotations - -from pathlib import Path -from typing import Literal - -from pydantic_settings import BaseSettings, SettingsConfigDict - - -class Settings(BaseSettings): - """PryScraper application settings.""" - - model_config = SettingsConfigDict( - env_file=".env", - env_file_encoding="utf-8", - case_sensitive=False, - extra="ignore", - ) - - # Core - app_name: str = "PryScraper" - app_version: str = "3.0.0" - environment: Literal["dev", "staging", "prod"] = "dev" - log_level: str = "INFO" - port: int = 8002 - - # Database - database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry" - - # Cache - redis_url: str = "redis://localhost:6379/0" - - # LLM Providers - openai_api_key: str = "" - anthropic_api_key: str = "" - cohere_api_key: str = "" - ollama_url: str = "http://localhost:11434" - - # Stealth - stealth_enabled: bool = True - random_user_agent: bool = True - min_delay_ms: int = 500 - max_delay_ms: int = 3000 - - # Rate limiting - rate_limit_per_domain: int = 10 # requests per second - rate_limit_per_ip: int = 60 - - # x402 Payment - x402_enabled: bool = False - x402_pay_to: str = "" - - # MCP - mcp_enabled: bool = True - mcp_tools_count: int = 8 - - # Storage - screenshot_dir: Path = Path("/tmp/pry-screenshots") # nosec B108 - cache_ttl_seconds: int = 3600 - - -_settings: Settings | None = None - - -def get_settings() -> Settings: - """Get cached settings instance.""" - global _settings - if _settings is None: - _settings = Settings() - return _settings diff --git a/deps.py b/deps.py index 93cdb00..d1c0e2e 100644 --- a/deps.py +++ b/deps.py @@ -6,8 +6,6 @@ direct access to shared state. Usage: from deps import scraper, cache, automator, ... - -Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper """ # SPDX-License-Identifier: MIT @@ -19,18 +17,16 @@ from automator import PryAutomator from cache import ResponseCache from extractor import SchemaExtractor from jobqueue import JobQueue -from mconfig import PryConfig from parser import DocumentParser from ratelimit import RateLimiter from scraper import PryScraper - -config = PryConfig() +from settings import settings scraper = PryScraper() automator = PryAutomator() parser = DocumentParser() extractor = SchemaExtractor() cache = ResponseCache(capacity=1000) -ratelimiter = RateLimiter(default_rpm=120, burst=200) +ratelimiter = RateLimiter(default_rpm=settings.rate_limit_rpm, burst=200) queue = JobQueue() advanced = PryAdvanced(cache=cache) diff --git a/mconfig.py b/mconfig.py deleted file mode 100644 index f39fbc2..0000000 --- a/mconfig.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Pry Config — runtime configuration for proxy, Tor, VPN, and anti-detection. -Users can configure at startup (env vars) or at runtime (API calls).""" - -# SPDX-License-Identifier: MIT -# Copyright (c) 2026 Rug Munch Media LLC -# -# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper -# Licensed under MIT. See LICENSE. - -import json -import os - -CONFIG_FILE = "/app/config.json" -DEFAULT_CONFIG = { - "proxy": {"enabled": False, "type": "", "url": "", "username": "", "password": ""}, - "tor": {"enabled": False, "socks5_host": "tor", "socks5_port": 9050, "control_port": 9051}, - "rotation": { - "user_agent": "rotate", - "ip": "off", - "timing": "random", - "delay_range": [0.5, 3.0], - }, - "stealth": { - "webdriver_override": True, - "canvas_noise": True, - "webrtc_disable": True, - "geolocation_spoof": True, - }, - "retry": {"max_attempts": 3, "min_quality": 20, "backoff": "exponential"}, - "output": {"default_format": "markdown", "max_chars": 100000, "include_links": True}, - "rate_limit": {"rpm": 120, "burst": 200}, -} - - -class PryConfig: - """Configuration manager — loads from env vars, config file, and API overrides.""" - - def __init__(self): - self.config = json.loads(json.dumps(DEFAULT_CONFIG)) # Deep copy - self._load_env() - self._load_file() - self._resolve_proxy_chain() - - def _load_env(self): - """Environment variables override defaults.""" - if os.getenv("TOR_ENABLED", "").lower() in ("1", "true", "yes"): - self.config["tor"]["enabled"] = True - if os.getenv("TOR_SOCKS5_HOST"): - self.config["tor"]["socks5_host"] = os.getenv("TOR_SOCKS5_HOST") - if os.getenv("TOR_SOCKS5_PORT"): - self.config["tor"]["socks5_port"] = int(os.getenv("TOR_SOCKS5_PORT")) - if os.getenv("PROXY_URL"): - self.config["proxy"]["enabled"] = True - self.config["proxy"]["url"] = os.getenv("PROXY_URL") - self.config["proxy"]["type"] = os.getenv("PROXY_TYPE", "http") - if os.getenv("PROXY_USERNAME"): - self.config["proxy"]["username"] = os.getenv("PROXY_USERNAME") - if os.getenv("PROXY_PASSWORD"): - self.config["proxy"]["password"] = os.getenv("PROXY_PASSWORD") - if os.getenv("IP_ROTATION"): - self.config["rotation"]["ip"] = os.getenv("IP_ROTATION") - if os.getenv("MAX_RETRIES"): - self.config["retry"]["max_attempts"] = int(os.getenv("MAX_RETRIES")) - if os.getenv("MIN_QUALITY"): - self.config["retry"]["min_quality"] = int(os.getenv("MIN_QUALITY")) - if os.getenv("RATE_LIMIT_RPM"): - self.config["rate_limit"]["rpm"] = int(os.getenv("RATE_LIMIT_RPM")) - - def _load_file(self): - if os.path.exists(CONFIG_FILE): - try: - with open(CONFIG_FILE) as f: - user_config = json.load(f) - self._deep_merge(self.config, user_config) - except OSError: - pass - - def _deep_merge(self, base: dict, override: dict): - for key, value in override.items(): - if key in base and isinstance(base[key], dict) and isinstance(value, dict): - self._deep_merge(base[key], value) - else: - base[key] = value - - def _resolve_proxy_chain(self): - """Build the proxy chain: Tor -> User Proxy -> Direct""" - proxies = [] - if self.config["tor"]["enabled"]: - proxies.append( - f"socks5://{self.config['tor']['socks5_host']}:{self.config['tor']['socks5_port']}" - ) - if self.config["proxy"]["enabled"]: - url = self.config["proxy"]["url"] - if self.config["proxy"]["username"]: - netloc = url.split("://")[1] if "://" in url else url - proxies.append( - f"{self.config['proxy']['type']}://{self.config['proxy']['username']}:{self.config['proxy']['password']}@{netloc}" - ) - else: - proxies.append(url) - self.config["_proxy_chain"] = proxies - self.config["_proxy_url"] = proxies[0] if proxies else None - - def get_proxy_url(self) -> str | None: - return self.config.get("_proxy_url") - - def get_proxy_chain(self) -> list: - return self.config.get("_proxy_chain", []) - - def get(self, key: str, default=None): - keys = key.split(".") - val = self.config - for k in keys: - if isinstance(val, dict): - val = val.get(k) - else: - return default - if val is None: - return default - return val if val is not None else default - - def to_dict(self) -> dict: - return {k: v for k, v in self.config.items() if not k.startswith("_")} - - def update(self, updates: dict) -> dict: - self._deep_merge(self.config, updates) - self._resolve_proxy_chain() - try: - with open(CONFIG_FILE, "w") as f: - json.dump(self.to_dict(), f, indent=2) - except OSError: - pass - return {"status": "ok", "config": self.to_dict()} diff --git a/routers/config.py b/routers/config.py index 85a4781..d027b65 100644 --- a/routers/config.py +++ b/routers/config.py @@ -1,6 +1,7 @@ -"""Pry — Config router (remaining api.py routes). +"""Pry — Config router (runtime configuration management). -Auto-extracted from api.py during the router-split refactor. +Provides endpoints to read and update the unified Pry settings object. +Runtime changes are persisted to the configured JSON file. """ # SPDX-License-Identifier: MIT @@ -13,37 +14,33 @@ from typing import Any from fastapi import APIRouter, Body -from mconfig import PryConfig +from settings import settings logger = logging.getLogger(__name__) router = APIRouter(tags=["Config"]) + @router.get("/v1/config", tags=["Config"], summary="Get current Pry configuration") async def get_config() -> dict[str, Any]: """Get current Pry configuration.""" - return {"success": True, "data": config.to_dict()} + return {"success": True, "data": settings.to_api_dict()} @router.post("/v1/config", tags=["Config"], summary="Update Pry configuration at runtime") async def update_config(updates: dict[str, Any] = Body(...)) -> dict[str, Any]: """Update Pry configuration at runtime.""" - result = config.update(updates) + result = settings.update(updates) return {"success": True, "data": result} @router.post("/v1/config/profile/tor", tags=["Config"], summary="Enable Tor routing for all requests") async def enable_tor() -> dict[str, Any]: """Enable Tor routing for all requests.""" - result = config.update( - {"tor": {"enabled": True}, "proxy": {"enabled": True, "url": "socks5://tor:9050"}} - ) + result = settings.enable_tor() resp_data = { "success": True, "data": result, "note": "Run 'docker compose --profile tor up -d' to start Tor container", } return resp_data - - -config = PryConfig() diff --git a/scraper.py b/scraper.py index 2872335..85ef206 100644 --- a/scraper.py +++ b/scraper.py @@ -19,10 +19,7 @@ import httpx import trafilatura from trafilatura.settings import use_config -from mconfig import PryConfig - -FLARESOLVERR_URL = "http://flaresolverr:8191/v1" -config = PryConfig() +from settings import settings USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", @@ -415,7 +412,7 @@ class PryScraper: return result async def _fetch_direct(self, url: str, headers: dict, timeout: int) -> str: - proxy_url = config.get_proxy_url() + proxy_url = settings.resolved_proxy_url async with httpx.AsyncClient( timeout=httpx.Timeout(timeout), follow_redirects=True, @@ -424,7 +421,7 @@ class PryScraper: cookies=httpx.Cookies(), proxy=proxy_url, ) as client: - delay = random.uniform(*config.get("rotation.delay_range", [0.5, 3.0])) + delay = random.uniform(*settings.rotation_delay_range) await asyncio.sleep(delay) resp = await client.get(url) resp.raise_for_status() @@ -433,7 +430,7 @@ class PryScraper: async def _fetch_via_flaresolverr(self, url: str, timeout: int) -> str: payload = {"cmd": "request.get", "url": url, "maxTimeout": timeout * 1000} async with httpx.AsyncClient(timeout=httpx.Timeout(timeout + 10)) as client: - resp = await client.post(FLARESOLVERR_URL, json=payload) + resp = await client.post(settings.flaresolverr_url, json=payload) resp.raise_for_status() data = resp.json() solution = data.get("solution", {}) diff --git a/settings.py b/settings.py index 8f432ff..53f0a50 100644 --- a/settings.py +++ b/settings.py @@ -1,14 +1,60 @@ """Pry — centralized settings via Pydantic BaseSettings. -Every env var lives here. Import `settings` singleton everywhere else.""" + +Every environment variable lives here. Import the `settings` singleton +everywhere else. + +Legacy env vars without the `PRY_` prefix (e.g. `PROXY_URL`, `TOR_ENABLED`) +are still accepted during a deprecation window. + +The settings object also loads and persists runtime overrides from a JSON +config file (default `/app/config.json`). +""" # SPDX-License-Identifier: MIT # Copyright (c) 2026 Rug Munch Media LLC # # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. +from __future__ import annotations +import json +from pathlib import Path +from typing import Any, Literal + +from pydantic import AliasChoices, Field from pydantic_settings import BaseSettings, SettingsConfigDict +DEFAULT_CONFIG_FILE = Path("/app/config.json") + +# Map legacy nested config keys (from the old mconfig JSON file and API) +# to flat attribute names on PrySettings. +_NESTED_KEY_MAP: dict[str, str] = { + "proxy.enabled": "proxy_enabled", + "proxy.type": "proxy_type", + "proxy.url": "proxy_url", + "proxy.username": "proxy_username", + "proxy.password": "proxy_password", + "tor.enabled": "tor_enabled", + "tor.socks5_host": "tor_socks5_host", + "tor.socks5_port": "tor_socks5_port", + "tor.control_port": "tor_control_port", + "rotation.user_agent": "rotation_user_agent", + "rotation.ip": "ip_rotation", + "rotation.timing": "rotation_timing", + "rotation.delay_range": "rotation_delay_range", + "stealth.webdriver_override": "webdriver_override", + "stealth.canvas_noise": "canvas_noise", + "stealth.webrtc_disable": "webrtc_disable", + "stealth.geolocation_spoof": "geolocation_spoof", + "retry.max_attempts": "max_retries", + "retry.min_quality": "min_quality", + "retry.backoff": "retry_backoff", + "output.default_format": "default_format", + "output.max_chars": "max_chars", + "output.include_links": "include_links", + "rate_limit.rpm": "rate_limit_rpm", +} + class PrySettings(BaseSettings): model_config = SettingsConfigDict( @@ -17,45 +63,261 @@ class PrySettings(BaseSettings): env_file_encoding="utf-8", case_sensitive=False, extra="ignore", + populate_by_name=True, ) # Core - url: str = "http://localhost:8002" + app_name: str = "Pry" + app_version: str = "3.0.0" + environment: Literal["dev", "staging", "prod"] = "dev" + log_level: str = "INFO" host: str = "0.0.0.0" # nosec B104 port: int = 8002 - - # Ollama LLM endpoint - ollama_url: str = "http://100.104.130.92:11434" - - # FlareSolverr Cloudflare bypass endpoint - flaresolverr_url: str = "http://flaresolverr:8191/v1" - - # Webhook secret for job callbacks - webhook_secret: str = "pry-webhook-secret" - - # API key for endpoint authentication (empty = disabled) + url: str = "http://localhost:8002" api_key: str = "" - # OpenRouter API key (optional, for vision models) - openrouter_api_key: str = "" - - # Proxy / Tor - proxy_url: str = "" - proxy_type: str = "http" - proxy_username: str = "" - proxy_password: str = "" - tor_enabled: bool = False - tor_socks5_host: str = "tor" - tor_socks5_port: int = 9050 - - # Rotation - ip_rotation: str = "" - max_retries: int = 0 - min_quality: int = 0 - rate_limit_rpm: int = 0 - - # Redis (optional, for job queue persistence) + # Database / cache + database_url: str = "postgresql+asyncpg://pry:pry@localhost/pry" redis_url: str = "redis://localhost:6379/0" + # LLM providers + ollama_url: str = "http://100.104.130.92:11434" + openrouter_api_key: str = "" + openai_api_key: str = "" + anthropic_api_key: str = "" + cohere_api_key: str = "" + # Services + flaresolverr_url: str = "http://flaresolverr:8191/v1" + webhook_secret: str = "pry-webhook-secret" + + # Proxy (legacy env aliases for migration) + proxy_url: str = Field( + default="", + validation_alias=AliasChoices("PRY_PROXY_URL", "PROXY_URL"), + ) + proxy_type: str = Field( + default="http", + validation_alias=AliasChoices("PRY_PROXY_TYPE", "PROXY_TYPE"), + ) + proxy_username: str = Field( + default="", + validation_alias=AliasChoices("PRY_PROXY_USERNAME", "PROXY_USERNAME"), + ) + proxy_password: str = Field( + default="", + validation_alias=AliasChoices("PRY_PROXY_PASSWORD", "PROXY_PASSWORD"), + ) + proxy_enabled: bool = Field( + default=False, + validation_alias=AliasChoices("PRY_PROXY_ENABLED", "PROXY_ENABLED"), + ) + + # Tor (legacy env aliases) + tor_enabled: bool = Field( + default=False, + validation_alias=AliasChoices("PRY_TOR_ENABLED", "TOR_ENABLED"), + ) + tor_socks5_host: str = Field( + default="tor", + validation_alias=AliasChoices("PRY_TOR_SOCKS5_HOST", "TOR_SOCKS5_HOST"), + ) + tor_socks5_port: int = Field( + default=9050, + validation_alias=AliasChoices("PRY_TOR_SOCKS5_PORT", "TOR_SOCKS5_PORT"), + ) + tor_control_port: int = 9051 + + # Rotation / retry (legacy env aliases) + rotation_user_agent: str = "rotate" + ip_rotation: str = Field( + default="off", + validation_alias=AliasChoices("PRY_IP_ROTATION", "IP_ROTATION"), + ) + rotation_timing: str = "random" + rotation_delay_range: tuple[float, float] = (0.5, 3.0) + max_retries: int = Field( + default=3, + validation_alias=AliasChoices("PRY_MAX_RETRIES", "MAX_RETRIES"), + ) + min_quality: int = Field( + default=20, + validation_alias=AliasChoices("PRY_MIN_QUALITY", "MIN_QUALITY"), + ) + retry_backoff: str = "exponential" + rate_limit_rpm: int = Field( + default=120, + validation_alias=AliasChoices("PRY_RATE_LIMIT_RPM", "RATE_LIMIT_RPM"), + ) + + # Stealth + stealth_enabled: bool = True + random_user_agent: bool = True + webdriver_override: bool = True + canvas_noise: bool = True + webrtc_disable: bool = True + geolocation_spoof: bool = True + min_delay_ms: int = 500 + max_delay_ms: int = 3000 + + # Output + default_format: str = "markdown" + max_chars: int = 100000 + include_links: bool = True + + # Storage + screenshot_dir: Path = Path("/tmp/pry-screenshots") # nosec B108 + cache_ttl_seconds: int = 3600 + config_file: Path = Field( + default=DEFAULT_CONFIG_FILE, + validation_alias=AliasChoices("PRY_CONFIG_FILE", "CONFIG_FILE"), + ) + + # x402 / MCP + x402_enabled: bool = False + x402_pay_to: str = "" + mcp_enabled: bool = True + mcp_tools_count: int = 8 + + # Rate limits (legacy Settings class) + rate_limit_per_domain: int = 10 + rate_limit_per_ip: int = 60 + + def model_post_init(self, __context: Any) -> None: + """Load runtime overrides from the JSON config file after env parsing.""" + self._load_config_file() + + def _load_config_file(self) -> None: + path = self.config_file + if not path.exists(): + return + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + return + if not isinstance(data, dict): + return + self._apply_nested_updates(data) + + def _apply_nested_updates(self, data: dict[str, Any]) -> None: + """Flatten nested config updates and set attributes on self.""" + flat: dict[str, Any] = {} + + def _flatten(obj: Any, prefix: str = "") -> None: + if isinstance(obj, dict): + for key, value in obj.items(): + new_prefix = f"{prefix}.{key}" if prefix else key + _flatten(value, new_prefix) + else: + flat[prefix] = obj + + _flatten(data) + for nested_key, value in flat.items(): + attr = _NESTED_KEY_MAP.get(nested_key, nested_key) + if hasattr(self, attr): + if value is None: + continue + current = getattr(self, attr) + expected_type = type(current) + try: + if expected_type is tuple and isinstance(value, list): + setattr(self, attr, tuple(value)) + else: + setattr(self, attr, expected_type(value)) + except (TypeError, ValueError): + setattr(self, attr, value) + + @property + def resolved_proxy_chain(self) -> list[str]: + """Return proxy chain: Tor -> user proxy -> direct.""" + proxies: list[str] = [] + if self.tor_enabled: + proxies.append(f"socks5://{self.tor_socks5_host}:{self.tor_socks5_port}") + if self.proxy_enabled and self.proxy_url: + if self.proxy_username: + netloc = self.proxy_url.split("://")[1] if "://" in self.proxy_url else self.proxy_url + proxies.append( + f"{self.proxy_type}://{self.proxy_username}:{self.proxy_password}@{netloc}" + ) + else: + proxies.append(self.proxy_url) + return proxies + + @property + def resolved_proxy_url(self) -> str | None: + """Return the first proxy in the chain, or None for direct connection.""" + chain = self.resolved_proxy_chain + return chain[0] if chain else None + + def to_api_dict(self) -> dict[str, Any]: + """Return the nested config representation used by the /v1/config API.""" + return { + "proxy": { + "enabled": self.proxy_enabled, + "type": self.proxy_type, + "url": self.proxy_url, + "username": self.proxy_username, + "password": self.proxy_password, + }, + "tor": { + "enabled": self.tor_enabled, + "socks5_host": self.tor_socks5_host, + "socks5_port": self.tor_socks5_port, + "control_port": self.tor_control_port, + }, + "rotation": { + "user_agent": self.rotation_user_agent, + "ip": self.ip_rotation, + "timing": self.rotation_timing, + "delay_range": list(self.rotation_delay_range), + }, + "stealth": { + "webdriver_override": self.webdriver_override, + "canvas_noise": self.canvas_noise, + "webrtc_disable": self.webrtc_disable, + "geolocation_spoof": self.geolocation_spoof, + }, + "retry": { + "max_attempts": self.max_retries, + "min_quality": self.min_quality, + "backoff": self.retry_backoff, + }, + "output": { + "default_format": self.default_format, + "max_chars": self.max_chars, + "include_links": self.include_links, + }, + "rate_limit": { + "rpm": self.rate_limit_rpm, + }, + } + + def update(self, updates: dict[str, Any]) -> dict[str, Any]: + """Apply nested runtime config updates and persist to disk.""" + self._apply_nested_updates(updates) + self._persist() + return {"status": "ok", "config": self.to_api_dict()} + + def _persist(self) -> None: + """Write the current config to the configured JSON file.""" + path = self.config_file + try: + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(self.to_api_dict(), f, indent=2) + except OSError: + pass + + def enable_tor(self) -> dict[str, Any]: + """Convenience helper used by the /v1/config/profile/tor endpoint.""" + return self.update( + { + "tor": {"enabled": True}, + "proxy": {"enabled": True, "url": "socks5://tor:9050"}, + } + ) + + +# Public singleton. Modules should import this directly. settings = PrySettings() diff --git a/tests/test_mconfig.py b/tests/test_mconfig.py deleted file mode 100644 index 33ce8fe..0000000 --- a/tests/test_mconfig.py +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-License-Identifier: MIT -# Copyright (c) 2026 Rug Munch Media LLC -# -# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper -# Licensed under MIT. See LICENSE. -"""Tests for Pry config.""" - -from mconfig import PryConfig - - -def test_config_get_default() -> None: - c = PryConfig() - assert c.get("nonexistent.key", "fallback") == "fallback" - - -def test_config_get_retry_exists() -> None: - c = PryConfig() - val = c.get("retry.max_attempts") - assert isinstance(val, int) - assert val > 0 - - -def test_config_get_rate_limit_exists() -> None: - c = PryConfig() - val = c.get("rate_limit.rpm") - assert isinstance(val, int) - assert val > 0 - - -def test_config_to_dict_excludes_private() -> None: - c = PryConfig() - d = c.to_dict() - assert "_proxy_chain" not in d - assert "_proxy_url" not in d - - -def test_config_update_returns_ok() -> None: - c = PryConfig() - result = c.update({"retry": {"max_attempts": 5}}) - assert result["status"] == "ok" - assert c.get("retry.max_attempts") == 5 - - -def test_config_get_proxy_url_string_or_none() -> None: - c = PryConfig() - url = c.get_proxy_url() - assert url is None or isinstance(url, str) diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..74dd731 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. See LICENSE. +"""Tests for unified Pry settings.""" + +import json +from pathlib import Path + +import pytest + +from settings import _NESTED_KEY_MAP, PrySettings + + +@pytest.fixture +def isolated_settings(tmp_path: Path) -> PrySettings: + """Return a fresh settings instance with a temp config file.""" + config_file = tmp_path / "config.json" + return PrySettings(config_file=config_file) + + +def test_settings_get_default(isolated_settings: PrySettings) -> None: + assert isolated_settings.max_retries == 3 + assert isolated_settings.rate_limit_rpm == 120 + + +def test_settings_proxy_resolution(isolated_settings: PrySettings) -> None: + assert isolated_settings.resolved_proxy_url is None + isolated_settings.proxy_enabled = True + isolated_settings.proxy_url = "http://proxy.example:8080" + assert isolated_settings.resolved_proxy_url == "http://proxy.example:8080" + + +def test_settings_tor_chain(isolated_settings: PrySettings) -> None: + isolated_settings.tor_enabled = True + isolated_settings.proxy_enabled = True + isolated_settings.proxy_url = "http://proxy.example:8080" + chain = isolated_settings.resolved_proxy_chain + assert len(chain) == 2 + assert chain[0].startswith("socks5://") + + +def test_settings_to_api_dict_excludes_private(isolated_settings: PrySettings) -> None: + data = isolated_settings.to_api_dict() + assert "_proxy_chain" not in data + assert "proxy" in data + assert isinstance(data["proxy"]["enabled"], bool) + + +def test_settings_update_and_persist(isolated_settings: PrySettings) -> None: + result = isolated_settings.update({"retry": {"max_attempts": 5}}) + assert result["status"] == "ok" + assert isolated_settings.max_retries == 5 + assert isolated_settings.config_file.exists() + saved = json.loads(isolated_settings.config_file.read_text(encoding="utf-8")) + assert saved["retry"]["max_attempts"] == 5 + + +def test_settings_load_config_file(tmp_path: Path) -> None: + config_file = tmp_path / "config.json" + config_file.write_text( + json.dumps({"retry": {"max_attempts": 7}, "tor": {"enabled": True}}), + encoding="utf-8", + ) + s = PrySettings(config_file=config_file) + assert s.max_retries == 7 + assert s.tor_enabled is True + + +def test_settings_legacy_env_vars(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("PROXY_URL", "http://legacy-proxy:8080") + monkeypatch.setenv("TOR_ENABLED", "true") + monkeypatch.setenv("MAX_RETRIES", "9") + config_file = tmp_path / "config.json" + s = PrySettings(config_file=config_file) + assert s.proxy_url == "http://legacy-proxy:8080" + assert s.tor_enabled is True + assert s.max_retries == 9 + + +def test_settings_enable_tor(isolated_settings: PrySettings) -> None: + result = isolated_settings.enable_tor() + assert result["status"] == "ok" + assert isolated_settings.tor_enabled is True + assert isolated_settings.proxy_enabled is True + assert isolated_settings.proxy_url == "socks5://tor:9050" + + +def test_nested_key_map_complete() -> None: + """Ensure every API-facing nested key maps to a settings attribute.""" + for attr in _NESTED_KEY_MAP.values(): + assert attr in PrySettings.model_fields, f"PrySettings missing {attr}"