feat(url_guard): SSRF and path traversal guard for scraper inputs

- Add url_guard.py with validate_url() that rejects:
  - Non-HTTP schemes (file://, ftp://, javascript:)
  - Private/reserved IP ranges (RFC 1918, loopback, CGNAT, link-local)
  - Internal hostnames (localhost, kubernetes, docker, metadata, etc.)
  - Embedded credentials in URLs
  - Path traversal sequences (.., %2e%2e%2f, %%2e%%2e%%2f)
- Integrate into scraper.scrape() and UltimateScraper.scrape()
- Use typed InvalidRequestError for structured API error responses
- Add 24 tests covering all rejection cases and valid URLs
- 560 tests passing, ruff clean
This commit is contained in:
Crypto Rug Munch 2026-07-03 12:00:40 +02:00
parent da31a1f9e7
commit 2970c15dbb
4 changed files with 275 additions and 0 deletions

View file

@ -19,8 +19,10 @@ import httpx
import trafilatura
from trafilatura.settings import use_config
from errors import InvalidRequestError
from settings import settings
from stealth_engine import StealthEngine
from url_guard import validate_url
USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
@ -281,6 +283,10 @@ class PryScraper:
async def scrape(self, url: str, options: dict | None = None) -> dict[str, Any]:
if not url or not url.startswith(("http://", "https://")):
return {"status": "error", "url": url, "error": "Invalid URL", "content": ""}
try:
validate_url(url)
except InvalidRequestError as e:
return {"status": "error", "url": url, "error": str(e), "content": ""}
opts = options or {}