feat(scraper): wire orphaned anti-detection modules into fallback chain
- Integrate TLSScraper (curl_cffi TLS fingerprint impersonation) as Tier 2. - Integrate CamoufoxBrowser as Tier 6 stealth Firefox alternative. - Inject StealthEngine.generate_all() into Playwright contexts in both ultimate_scraper.py and scraper.py. - Update UltimateScraper docstring to 12-tier fallback chain. - Add 18 tests covering TLS, Camoufox, full fallthrough, and tier success paths. - All 518 tests pass; ruff clean.
This commit is contained in:
parent
1f9e71d294
commit
775f5412bf
3 changed files with 219 additions and 35 deletions
14
scraper.py
14
scraper.py
|
|
@ -20,6 +20,7 @@ import trafilatura
|
|||
from trafilatura.settings import use_config
|
||||
|
||||
from settings import settings
|
||||
from stealth_engine import StealthEngine
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
||||
|
|
@ -467,17 +468,8 @@ class PryScraper:
|
|||
geolocation={"latitude": 40.7128, "longitude": -74.0060},
|
||||
)
|
||||
# Stealth: override automation indicators
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||||
window.chrome = { runtime: {} };
|
||||
// Override permissions query to avoid automation detection
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (p) => (
|
||||
p.name === 'notifications' ? Promise.resolve({state: 'denied'}) : originalQuery(p)
|
||||
);
|
||||
""")
|
||||
stealth_script = StealthEngine().generate_all()
|
||||
await context.add_init_script(stealth_script)
|
||||
page = await context.new_page()
|
||||
# Human-like navigation: random wait before page load
|
||||
await asyncio.sleep(random.uniform(0.5, 2.0))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue