feat(url_guard): SSRF and path traversal guard for scraper inputs
- Add url_guard.py with validate_url() that rejects: - Non-HTTP schemes (file://, ftp://, javascript:) - Private/reserved IP ranges (RFC 1918, loopback, CGNAT, link-local) - Internal hostnames (localhost, kubernetes, docker, metadata, etc.) - Embedded credentials in URLs - Path traversal sequences (.., %2e%2e%2f, %%2e%%2e%%2f) - Integrate into scraper.scrape() and UltimateScraper.scrape() - Use typed InvalidRequestError for structured API error responses - Add 24 tests covering all rejection cases and valid URLs - 560 tests passing, ruff clean
This commit is contained in:
parent
da31a1f9e7
commit
2970c15dbb
4 changed files with 275 additions and 0 deletions
|
|
@ -18,7 +18,9 @@ import time
|
|||
from typing import Any, ClassVar
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from errors import InvalidRequestError
|
||||
from resilience import registry
|
||||
from url_guard import validate_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -111,6 +113,11 @@ class UltimateScraper:
|
|||
Will try up to 10 strategies until one succeeds."""
|
||||
opts = options or {}
|
||||
errors: list[str] = []
|
||||
try:
|
||||
validate_url(url)
|
||||
except InvalidRequestError as e:
|
||||
return {"status": "error", "url": url, "error": str(e)}
|
||||
|
||||
|
||||
proxy_url = self._resolve_proxy_url(opts)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue