cryptorugmunch
2970c15dbb
- Add url_guard.py with validate_url() that rejects: - Non-HTTP schemes (file://, ftp://, javascript:) - Private/reserved IP ranges (RFC 1918, loopback, CGNAT, link-local) - Internal hostnames (localhost, kubernetes, docker, metadata, etc.) - Embedded credentials in URLs - Path traversal sequences (.., %2e%2e%2f, %%2e%%2e%%2f) - Integrate into scraper.scrape() and UltimateScraper.scrape() - Use typed InvalidRequestError for structured API error responses - Add 24 tests covering all rejection cases and valid URLs - 560 tests passing, ruff clean
107 lines
3.7 KiB
Python
107 lines
3.7 KiB
Python
"""Tests for SSRF and path-traversal guard."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from errors import InvalidRequestError
|
|
from url_guard import validate_url
|
|
|
|
|
|
class TestValidateURL:
|
|
def test_accepts_normal_https(self):
|
|
result = validate_url("https://example.com/page")
|
|
assert result == "https://example.com/page"
|
|
|
|
def test_accepts_normal_http(self):
|
|
result = validate_url("http://example.com/page")
|
|
assert result == "http://example.com/page"
|
|
|
|
def test_accepts_https_with_path_and_query(self):
|
|
url = "https://api.example.com/v1/data?key=val&page=2#section"
|
|
result = validate_url(url)
|
|
assert result == url
|
|
|
|
def test_rejects_empty_url(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("")
|
|
|
|
def test_rejects_ftp_scheme(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("ftp://example.com/file")
|
|
|
|
def test_rejects_file_scheme(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("file:///etc/passwd")
|
|
|
|
def test_rejects_javascript_scheme(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("javascript:alert(1)")
|
|
|
|
def test_rejects_localhost_hostname(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://localhost:8080/admin")
|
|
|
|
def test_rejects_localhost_ip(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://127.0.0.1:8080/admin")
|
|
|
|
def test_rejects_private_ip_10_dot(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://10.0.0.1/admin")
|
|
|
|
def test_rejects_private_ip_192_168(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://192.168.1.1/admin")
|
|
|
|
def test_rejects_private_ip_172(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://172.16.0.1/admin")
|
|
|
|
def test_rejects_internal_hostname(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://internal.corp/admin")
|
|
|
|
def test_rejects_kubernetes_metadata(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://kubernetes.default.svc.cluster.local")
|
|
|
|
def test_rejects_embedded_credentials(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://user:pass@example.com/")
|
|
|
|
def test_rejects_path_traversal_simple(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("https://example.com/../../etc/passwd")
|
|
|
|
def test_rejects_path_traversal_encoded(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("https://example.com/%2e%2e%2fetc/passwd")
|
|
|
|
def test_rejects_path_traversal_double_encoded(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("https://example.com/%252e%252e%252fetc")
|
|
|
|
def test_rejects_tailscale_ip(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://100.64.0.1/")
|
|
|
|
def test_rejects_empty_hostname(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http:///path")
|
|
|
|
def test_rejects_docker_host(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://docker/container")
|
|
|
|
def test_rejects_metadata_google(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://metadata.google.internal/")
|
|
|
|
def test_rejects_link_local_ipv6(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://[fe80::1]/")
|
|
|
|
def test_rejects_loopback_ipv6(self):
|
|
with pytest.raises(InvalidRequestError):
|
|
validate_url("http://[::1]/")
|