440 lines
18 KiB
Python
440 lines
18 KiB
Python
"""Pry — Ultimate Anti-Detection Scraper.
|
|
Multi-strategy scraping with 10+ bypass methods and intelligent fallback.
|
|
Beats Cloudflare, DataDome, Akamai, Imperva, PerimeterX, and generic blocks."""
|
|
|
|
# SPDX-License-Identifier: BSL-1.1
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — Stealth / Anti-Detection Module
|
|
# Licensed under Business Source License 1.1 — see LICENSE-BSL-STEALTH.
|
|
# Change Date: 2029-01-01 (converts to MIT).
|
|
|
|
import importlib.util
|
|
import logging
|
|
import os
|
|
import random
|
|
import re
|
|
from typing import Any, ClassVar
|
|
from urllib.parse import urlparse
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Try importing optional bypass libraries
|
|
_has_cloudscraper = importlib.util.find_spec("cloudscraper") is not None
|
|
_has_undetected = importlib.util.find_spec("undetected_chromedriver") is not None
|
|
_has_playwright = importlib.util.find_spec("playwright") is not None
|
|
_has_tor = importlib.util.find_spec("aiohttp_socks") is not None
|
|
|
|
if _has_playwright:
|
|
from playwright.async_api import async_playwright
|
|
|
|
|
|
class UltimateScraper:
|
|
"""10-tier anti-detection scraper with automatic fallback.
|
|
|
|
Tiers:
|
|
1. Direct HTTP (smart headers, rotating UAs)
|
|
2. cloudscraper (Python-native Cloudflare bypass)
|
|
3. FlareSolverr (Cloudflare/WAF bypass service)
|
|
4. undetected-chromedriver (modified Chrome, no detection)
|
|
5. Playwright stealth (full browser with human behavior)
|
|
6. Googlebot UA (search engine crawl mimic)
|
|
7. Tor proxy (anonymous routing via SOCKS5)
|
|
8. Archive.org / Wayback Machine (cached version)
|
|
9. Google Cache (cached version)
|
|
10. Textise dot iitty (text-only version)
|
|
"""
|
|
|
|
USER_AGENTS: ClassVar[list[str]] = [
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
|
|
]
|
|
|
|
def __init__(self) -> None:
|
|
self._ua_index = 0
|
|
|
|
def _rotate_ua(self) -> str:
|
|
ua = self.USER_AGENTS[self._ua_index % len(self.USER_AGENTS)]
|
|
self._ua_index += 1
|
|
return ua
|
|
|
|
def _build_headers(self, url: str) -> dict[str, str]:
|
|
parsed = urlparse(url)
|
|
return {
|
|
"User-Agent": self._rotate_ua(),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": random.choice(
|
|
["en-US,en;q=0.9", "en-GB,en;q=0.8", "en-US,en;q=0.7"]
|
|
),
|
|
"Referer": f"https://{parsed.netloc}/",
|
|
"DNT": "1",
|
|
"Connection": "keep-alive",
|
|
"Upgrade-Insecure-Requests": "1",
|
|
"Sec-Fetch-Dest": "document",
|
|
"Sec-Fetch-Mode": "navigate",
|
|
"Sec-Fetch-Site": "none",
|
|
"Sec-Fetch-User": "?1",
|
|
}
|
|
|
|
async def scrape(self, url: str, options: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
"""Scrape any URL with automatic bypass strategy selection.
|
|
Will try up to 10 strategies until one succeeds."""
|
|
opts = options or {}
|
|
errors: list[str] = []
|
|
|
|
proxy_url = self._resolve_proxy_url(opts)
|
|
|
|
# Tier 0: Premium proxy (if configured and forced or first attempt)
|
|
if proxy_url and opts.get("use_premium_proxy_first", True):
|
|
html = await self._tier_premium_proxy(url, proxy_url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "premium_proxy")
|
|
errors.append("premium_proxy_failed")
|
|
|
|
# Tier 1: Direct HTTP
|
|
html = await self._tier_direct(url, opts, proxy_url=proxy_url)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "direct")
|
|
|
|
# Tier 2: cloudscraper (Python Cloudflare bypass)
|
|
html = await self._tier_cloudscraper(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "cloudscraper")
|
|
|
|
# Tier 3: FlareSolverr
|
|
html = await self._tier_flaresolverr(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "flaresolverr")
|
|
|
|
# Tier 4: undetected-chromedriver
|
|
html = await self._tier_undetected(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "undetected")
|
|
|
|
# Tier 5: Playwright stealth
|
|
html = await self._tier_playwright(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "playwright")
|
|
|
|
# Tier 6: Googlebot UA
|
|
html = await self._tier_googlebot(url, opts, proxy_url=proxy_url)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "googlebot")
|
|
|
|
# Tier 7: Premium proxy retry (after cheap tiers fail)
|
|
if proxy_url and not opts.get("use_premium_proxy_first", True):
|
|
html = await self._tier_premium_proxy(url, proxy_url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "premium_proxy")
|
|
errors.append("premium_proxy_failed")
|
|
|
|
# Tier 8: Archive.org / Wayback Machine
|
|
html = await self._tier_archive(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "archive")
|
|
|
|
# Tier 9: Google Cache
|
|
html = await self._tier_google_cache(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "google_cache")
|
|
|
|
# Tier 10: Tor (anonymous routing)
|
|
html = await self._tier_tor(url, opts)
|
|
if html and self._is_valid(html):
|
|
return self._result(url, html, "tor")
|
|
|
|
last_err = errors[-1] if errors else "all_tiers_failed"
|
|
return {
|
|
"status": "error",
|
|
"url": url,
|
|
"error": "; ".join(errors[-5:]) if errors else last_err,
|
|
"content": "",
|
|
"proxy_recommendation": self._recommend_proxy(last_err),
|
|
}
|
|
|
|
def _resolve_proxy_url(self, opts: dict[str, Any]) -> str | None:
|
|
"""Resolve the configured proxy URL from options, ProxyManager, or env."""
|
|
if opts.get("proxy_url"):
|
|
return opts["proxy_url"]
|
|
if opts.get("proxy") is False:
|
|
return None
|
|
try:
|
|
from proxy_manager import ProxyManager
|
|
|
|
pm = ProxyManager()
|
|
return pm.get_proxy_url()
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug("proxy_resolve_failed", extra={"error": str(e)[:80]})
|
|
env_url = os.getenv("PRY_PROXY_URL") or os.getenv("PROXY_URL")
|
|
return env_url or None
|
|
|
|
def _recommend_proxy(self, last_error: str) -> dict[str, Any] | None:
|
|
"""Build a proxy recommendation for the response when scraping fails."""
|
|
try:
|
|
from proxy_manager import ProxyManager
|
|
|
|
return ProxyManager().get_recommendation(last_error)
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug("proxy_recommend_failed", extra={"error": str(e)[:80]})
|
|
return None
|
|
|
|
def _result(self, url: str, html: str, method: str) -> dict[str, Any]:
|
|
return {
|
|
"status": "ok",
|
|
"url": url,
|
|
"raw_html": html,
|
|
"method": method,
|
|
"content": self._extract_text(html),
|
|
}
|
|
|
|
def _is_valid(self, html: str | None) -> bool:
|
|
if not html or len(html) < 500:
|
|
return False
|
|
lower = html.lower()
|
|
block_indicators = [
|
|
"cloudflare",
|
|
"attention required",
|
|
"just a moment",
|
|
"enable javascript",
|
|
"verify you are human",
|
|
]
|
|
if any(i in lower for i in block_indicators) and len(html) < 5000:
|
|
return False
|
|
body = re.search(r"<body[^>]*>(.*?)</body>", html, re.DOTALL)
|
|
if body:
|
|
text = re.sub(r"<[^>]+>", " ", body.group(1)).strip()
|
|
if len(text) < 50:
|
|
return False
|
|
return True
|
|
|
|
def _extract_text(self, html: str) -> str:
|
|
"""Extract readable text from HTML."""
|
|
try:
|
|
from trafilatura import extract
|
|
|
|
content = extract(
|
|
html, output_format="markdown", include_links=True, include_images=False
|
|
)
|
|
if content and len(content) > 100:
|
|
return content
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
try:
|
|
from readability import Document
|
|
|
|
doc = Document(html)
|
|
content = doc.summary()
|
|
if content and len(content) > 100:
|
|
from markdownify import markdownify
|
|
|
|
return markdownify(content, heading_arrows=False, strip=["script", "style"])
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
# Last resort: strip HTML tags
|
|
text = re.sub(r"<[^>]+>", " ", html)
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|
return text[:50000] if len(text) > 50000 else text
|
|
|
|
async def _tier_direct(
|
|
self, url: str, opts: dict[str, Any], proxy_url: str | None = None
|
|
) -> str | None:
|
|
try:
|
|
import httpx
|
|
|
|
async with httpx.AsyncClient(timeout=30, follow_redirects=True, proxy=proxy_url) as c:
|
|
r = await c.get(url, headers=self._build_headers(url))
|
|
if r.is_success:
|
|
return r.text
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
logger.debug("direct_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_premium_proxy(
|
|
self, url: str, proxy_url: str, opts: dict[str, Any]
|
|
) -> str | None:
|
|
"""Fetch via the configured premium residential/datacenter proxy."""
|
|
try:
|
|
import httpx
|
|
|
|
async with httpx.AsyncClient(timeout=45, follow_redirects=True, proxy=proxy_url) as c:
|
|
r = await c.get(url, headers=self._build_headers(url))
|
|
if r.is_success:
|
|
return r.text
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
logger.debug("premium_proxy_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_cloudscraper(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
if not _has_cloudscraper:
|
|
return None
|
|
try:
|
|
import cloudscraper
|
|
|
|
scraper = cloudscraper.create_scraper(
|
|
browser={"browser": "chrome", "platform": "windows", "mobile": False}
|
|
)
|
|
resp = scraper.get(url, timeout=30, headers={"User-Agent": self._rotate_ua()})
|
|
if resp.status_code == 200:
|
|
return resp.text
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug("cloudscraper_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_flaresolverr(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
from settings import settings
|
|
|
|
if not settings.flaresolverr_url:
|
|
return None
|
|
try:
|
|
import httpx
|
|
|
|
payload = {"cmd": "request.get", "url": url, "maxTimeout": 60000}
|
|
async with httpx.AsyncClient(timeout=60) as c:
|
|
r = await c.post(settings.flaresolverr_url, json=payload)
|
|
if r.is_success:
|
|
data = r.json()
|
|
solution = data.get("solution", {})
|
|
if solution.get("status") == 200:
|
|
return solution["response"]
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
logger.debug("flaresolverr_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_undetected(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
if not _has_undetected:
|
|
return None
|
|
try:
|
|
import asyncio
|
|
import time
|
|
|
|
def _do_fetch() -> str:
|
|
import undetected_chromedriver as uc
|
|
|
|
driver = uc.Chrome(headless=True, use_subprocess=True)
|
|
driver.get(url)
|
|
time.sleep(random.uniform(2, 4))
|
|
html = driver.page_source
|
|
driver.quit()
|
|
return html
|
|
|
|
return await asyncio.to_thread(_do_fetch)
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug("undetected_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_playwright(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
if not _has_playwright:
|
|
return None
|
|
try:
|
|
async with async_playwright() as pw:
|
|
browser = await pw.chromium.launch(
|
|
headless=True,
|
|
args=[
|
|
"--disable-blink-features=AutomationControlled",
|
|
"--no-sandbox",
|
|
"--disable-dev-shm-usage",
|
|
"--disable-web-security",
|
|
],
|
|
)
|
|
context = await browser.new_context(
|
|
viewport={
|
|
"width": random.choice([1920, 1366, 1440]),
|
|
"height": random.choice([1080, 768, 900]),
|
|
},
|
|
user_agent=self._rotate_ua(),
|
|
locale="en-US",
|
|
timezone_id="America/New_York",
|
|
)
|
|
await context.add_init_script("""
|
|
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
|
navigator.permissions.query = (p) => p.name === 'notifications' ? Promise.resolve({state: 'denied'}) : navigator.permissions.query(p);
|
|
""")
|
|
page = await context.new_page()
|
|
await page.goto(url, wait_until="domcontentloaded", timeout=60000)
|
|
await page.wait_for_timeout(random.randint(1000, 3000))
|
|
for _ in range(random.randint(2, 5)):
|
|
await page.evaluate(f"window.scrollBy(0, {random.randint(200, 500)})")
|
|
await page.wait_for_timeout(random.randint(300, 1000))
|
|
await page.mouse.move(random.randint(100, 1400), random.randint(100, 800))
|
|
html = await page.content()
|
|
await browser.close()
|
|
return html
|
|
except OSError as e:
|
|
logger.debug("playwright_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_googlebot(
|
|
self, url: str, opts: dict[str, Any], proxy_url: str | None = None
|
|
) -> str | None:
|
|
try:
|
|
import httpx
|
|
|
|
headers = self._build_headers(url)
|
|
headers["User-Agent"] = (
|
|
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
|
)
|
|
async with httpx.AsyncClient(timeout=30, follow_redirects=True, proxy=proxy_url) as c:
|
|
r = await c.get(url, headers=headers)
|
|
if r.is_success:
|
|
return r.text
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
logger.debug("googlebot_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_archive(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
"""Fetch from Wayback Machine CDX API for latest snapshot."""
|
|
try:
|
|
import httpx
|
|
|
|
cdx_url = f"https://web.archive.org/cdx/search/cdx?url={url}&output=json&limit=1&fl=timestamp,original"
|
|
async with httpx.AsyncClient(timeout=15) as c:
|
|
r = await c.get(cdx_url, headers={"User-Agent": "Mozilla/5.0"})
|
|
if r.is_success:
|
|
data = r.json()
|
|
if len(data) > 1:
|
|
timestamp = data[1][0]
|
|
archive_url = f"https://web.archive.org/web/{timestamp}id_/{url}"
|
|
ar = await c.get(archive_url, timeout=30, follow_redirects=True)
|
|
if ar.is_success:
|
|
return ar.text
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
logger.debug("archive_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_google_cache(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
try:
|
|
import httpx
|
|
|
|
cache_url = f"https://webcache.googleusercontent.com/search?q=cache:{url}"
|
|
async with httpx.AsyncClient(timeout=15, follow_redirects=True) as c:
|
|
r = await c.get(cache_url, headers={"User-Agent": "Mozilla/5.0"})
|
|
if r.is_success:
|
|
return r.text
|
|
except (httpx.HTTPError, httpx.RequestError) as e:
|
|
logger.debug("google_cache_failed", extra={"error": str(e)[:50]})
|
|
return None
|
|
|
|
async def _tier_tor(self, url: str, opts: dict[str, Any]) -> str | None:
|
|
"""Fetch via Tor SOCKS5 proxy."""
|
|
if not _has_tor:
|
|
return None
|
|
try:
|
|
import aiohttp
|
|
from aiohttp_socks import ProxyConnector
|
|
|
|
connector = ProxyConnector.from_url("socks5://127.0.0.1:9050")
|
|
async with (
|
|
aiohttp.ClientSession(connector=connector) as session,
|
|
session.get(
|
|
url,
|
|
headers=self._build_headers(url),
|
|
timeout=aiohttp.ClientTimeout(total=60),
|
|
) as resp,
|
|
):
|
|
if resp.status == 200:
|
|
return await resp.text()
|
|
except aiohttp.ClientError as e:
|
|
logger.debug("tor_failed", extra={"error": str(e)[:50]})
|
|
return None
|