feat(url_guard): SSRF and path traversal guard for scraper inputs
- Add url_guard.py with validate_url() that rejects: - Non-HTTP schemes (file://, ftp://, javascript:) - Private/reserved IP ranges (RFC 1918, loopback, CGNAT, link-local) - Internal hostnames (localhost, kubernetes, docker, metadata, etc.) - Embedded credentials in URLs - Path traversal sequences (.., %2e%2e%2f, %%2e%%2e%%2f) - Integrate into scraper.scrape() and UltimateScraper.scrape() - Use typed InvalidRequestError for structured API error responses - Add 24 tests covering all rejection cases and valid URLs - 560 tests passing, ruff clean
This commit is contained in:
parent
da31a1f9e7
commit
2970c15dbb
4 changed files with 275 additions and 0 deletions
|
|
@ -19,8 +19,10 @@ import httpx
|
|||
import trafilatura
|
||||
from trafilatura.settings import use_config
|
||||
|
||||
from errors import InvalidRequestError
|
||||
from settings import settings
|
||||
from stealth_engine import StealthEngine
|
||||
from url_guard import validate_url
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
|
|
@ -281,6 +283,10 @@ class PryScraper:
|
|||
async def scrape(self, url: str, options: dict | None = None) -> dict[str, Any]:
|
||||
if not url or not url.startswith(("http://", "https://")):
|
||||
return {"status": "error", "url": url, "error": "Invalid URL", "content": ""}
|
||||
try:
|
||||
validate_url(url)
|
||||
except InvalidRequestError as e:
|
||||
return {"status": "error", "url": url, "error": str(e), "content": ""}
|
||||
|
||||
opts = options or {}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue