pryscraper/proxy_manager.py

"""Pry — Proxy Manager with affiliate-signup flow.
Tries free proxies first (Tor, public rotating). When premium proxy is needed,
prompts user to sign up via affiliate links. Pre-wired to 5+ providers."""

import json
import logging
import os
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

logger = logging.getLogger(__name__)

PROXY_DIR = Path(os.path.expanduser("~/.pry/proxies"))
PROXY_DIR.mkdir(parents=True, exist_ok=True)

# Free proxy sources (no signup required)
FREE_PROXY_SOURCES = [
    {
        "name": "Tor",
        "type": "socks5",
        "url": "socks5://127.0.0.1:9050",
        "cost": "Free",
        "speed": "Slow",
        "reliability": "High",
    },
    {
        "name": "Public Pool (round-robin)",
        "type": "http",
        "url": "rotating",
        "cost": "Free",
        "speed": "Variable",
        "reliability": "Low",
    },
    {
        "name": "Cloudflare WARP",
        "type": "wireguard",
        "url": "warp://1.1.1.1",
        "cost": "Free",
        "speed": "Fast",
        "reliability": "High",
    },
]

# Premium proxy providers with full affiliate details
PREMIUM_PROXY_PROVIDERS = [
    {
        "name": "Bright Data",
        "tag": "brightdata",
        "url": "https://brightdata.com/?ref=pry",
        "signup_url": "https://brightdata.com/cp/start?aff_id=pry",
        "api_endpoint": "https://api.brightdata.com",
        "auth_format": "username:password",
        "commission": "$2-$50 per signup, up to 10% recurring",
        "cookie_days": 30,
        "free_trial": "Yes, $5 credit",
        "min_price": "$0.10/GB residential",
        "speed": "Very Fast",
        "reliability": "Very High",
        "ips": "72M+ residential, 770K+ datacenter",
        "geo_coverage": "195 countries",
        "protocols": ["HTTP", "SOCKS5", "HTTPS"],
        "note": "Industry leader. Best for high-volume scraping.",
    },
    {
        "name": "Smartproxy",
        "tag": "smartproxy",
        "url": "https://smartproxy.com/?ref=pry",
        "signup_url": "https://dashboard.smartproxy.com/registration?aff_id=pry",
        "api_endpoint": "https://api.smartproxy.com",
        "auth_format": "username:password",
        "commission": "20% recurring lifetime",
        "cookie_days": 60,
        "free_trial": "Yes, 100MB",
        "min_price": "$0.10/GB residential",
        "speed": "Fast",
        "reliability": "High",
        "ips": "40M+ residential",
        "geo_coverage": "195 countries",
        "protocols": ["HTTP", "SOCKS5"],
        "note": "Good balance of price and quality.",
    },
    {
        "name": "Oxylabs",
        "tag": "oxylabs",
        "url": "https://oxylabs.io/?ref=pry",
        "signup_url": "https://dashboard.oxylabs.io/?aff_id=pry",
        "api_endpoint": "https://api.oxylabs.com",
        "auth_format": "username:password",
        "commission": "Partner program (10-30% recurring)",
        "cookie_days": 30,
        "free_trial": "Yes, 5K requests",
        "min_price": "$1.20/GB residential",
        "speed": "Very Fast",
        "reliability": "Very High",
        "ips": "100M+ residential",
        "geo_coverage": "195 countries",
        "protocols": ["HTTP", "SOCKS5", "HTTPS"],
        "note": "Enterprise-grade. Best for big data scraping.",
    },
    {
        "name": "IPRoyal",
        "tag": "iproyal",
        "url": "https://iproyal.com/?ref=pry",
        "signup_url": "https://dashboard.iproyal.com/signup?aff=pry",
        "api_endpoint": "https://api.iproyal.com",
        "auth_format": "username:password",
        "commission": "30% recurring lifetime",
        "cookie_days": 60,
        "free_trial": "No",
        "min_price": "$0.04/GB residential",
        "speed": "Medium",
        "reliability": "High",
        "ips": "300K+ residential",
        "geo_coverage": "150+ countries",
        "protocols": ["HTTP", "SOCKS5"],
        "note": "Cheapest. Good for budget scraping.",
    },
    {
        "name": "Webshare",
        "tag": "webshare",
        "url": "https://www.webshare.io/?ref=pry",
        "signup_url": "https://proxy.webshare.io/register?aff=pry",
        "api_endpoint": "https://proxy.webshare.io/api/v2",
        "auth_format": "api_key",
        "commission": "30% recurring lifetime",
        "cookie_days": 60,
        "free_trial": "Yes, 10 free proxies",
        "min_price": "$0.03/proxy/month (datacenter)",
        "speed": "Very Fast",
        "reliability": "High",
        "ips": "500K+ datacenter/residential",
        "geo_coverage": "100+ countries",
        "protocols": ["HTTP", "SOCKS5"],
        "note": "Best free tier. Try before you buy.",
    },
    {
        "name": "Proxy-Seller",
        "tag": "proxyseller",
        "url": "https://proxy-seller.com/?ref=pry",
        "signup_url": "https://proxy-seller.com/?partner=pry",
        "auth_format": "username:password",
        "commission": "30% recurring",
        "cookie_days": 60,
        "free_trial": "No",
        "min_price": "$1.40/proxy/month",
        "speed": "Fast",
        "reliability": "High",
        "note": "Cheap private proxies.",
    },
    {
        "name": "NetNut",
        "tag": "netnut",
        "url": "https://netnut.io/?ref=pry",
        "signup_url": "https://netnut.io/?aff=pry",
        "auth_format": "username:password",
        "commission": "Partner program",
        "cookie_days": 30,
        "free_trial": "Yes, 7 days",
        "min_price": "$0.20/GB",
        "note": "Fast residential IPs from ISPs.",
    },
    {
        "name": "ProxyMesh",
        "tag": "proxymesh",
        "url": "https://proxymesh.com/?ref=pry",
        "signup_url": "https://proxymesh.com/?aff=pry",
        "auth_format": "username:password",
        "commission": "20% recurring",
        "cookie_days": 30,
        "free_trial": "No",
        "min_price": "$0.80/proxy",
        "note": "Simple rotating proxies.",
    },
    {
        "name": "Decodo (Smartproxy)",
        "tag": "decodo",
        "url": "https://decodo.com/?ref=pry",
        "signup_url": "https://decodo.com/signup?aff=pry",
        "auth_format": "username:password",
        "commission": "20% recurring",
        "cookie_days": 60,
        "note": "Budget option from Smartproxy.",
    },
    {
        "name": "PacketStream",
        "tag": "packetstream",
        "url": "https://packetstream.io/?ref=pry",
        "signup_url": "https://packetstream.io/signup?aff=pry",
        "auth_format": "username:password",
        "commission": "20% revenue share",
        "cookie_days": 60,
        "free_trial": "Pay-as-you-go",
        "min_price": "$0.05/GB",
        "note": "Bandwidth-sharing residential proxies.",
    },
]


@dataclass
class ProxyConfig:
    """Proxy configuration."""

    provider: str = "free"
    proxy_url: str = ""
    username: str = ""
    password: str = ""
    api_key: str = ""
    proxy_type: str = "http"
    geo: str = "auto"
    rotation: str = "per_request"
    auto_rotate: bool = True


class ProxyManager:
    """Manages proxy selection, free fallbacks, and premium signup flow."""

    def __init__(self, data_dir: Path | None = None) -> None:
        self.data_dir = data_dir or PROXY_DIR
        self.data_dir.mkdir(parents=True, exist_ok=True)
        self.config_file = self.data_dir / "active_config.json"
        self.creds_file = self.data_dir / "credentials.json"
        self.active_config = self._load_config()
        self.credentials: dict[str, ProxyConfig] = self._load_credentials()
        self.auto_configure_from_env()

    def _load_config(self) -> ProxyConfig:
        if self.config_file.exists():
            try:
                return ProxyConfig(**json.loads(self.config_file.read_text()))
            except (json.JSONDecodeError, TypeError):
                pass
        return ProxyConfig()

    def _load_credentials(self) -> dict[str, ProxyConfig]:
        if self.creds_file.exists():
            try:
                data = json.loads(self.creds_file.read_text())
                return {k: ProxyConfig(**v) for k, v in data.items()}
            except (json.JSONDecodeError, TypeError):
                pass
        return {}

    def _save_config(self) -> None:
        try:
            self.config_file.write_text(json.dumps(self.active_config.__dict__, indent=2))
        except OSError as e:
            logger.debug("proxy_config_save_failed", extra={"error": str(e)[:80]})

    def _save_credentials(self) -> None:
        try:
            data = {k: v.__dict__ for k, v in self.credentials.items()}
            self.creds_file.write_text(json.dumps(data, indent=2, default=str))
        except OSError as e:
            logger.debug("proxy_creds_save_failed", extra={"error": str(e)[:80]})

    def get_proxy_url(self) -> str | None:
        """Get the active proxy URL. Returns None if no proxy configured."""
        c = self.active_config
        if c.provider == "free":
            if c.proxy_url and c.proxy_url != "rotating":
                return c.proxy_url
            return None
        if c.provider not in self.credentials:
            return None
        cred = self.credentials[c.provider]
        if cred.api_key:
            return f"http://{cred.api_key}@proxy.{cred.provider}.com:8000"
        if cred.username and cred.password:
            return f"{cred.proxy_type}://{cred.username}:{cred.password}@{cred.proxy_url}"
        if cred.proxy_url:
            return cred.proxy_url
        return None

    def test_proxy(
        self, proxy_url: str, test_url: str = "https://httpbin.org/ip", timeout: int = 10
    ) -> dict[str, Any]:
        """Test a proxy and return its public IP, latency, and working status."""
        import httpx

        start = time.time()
        try:
            with httpx.Client(proxy=proxy_url, timeout=timeout) as c:
                r = c.get(test_url)
                elapsed = time.time() - start
                return {
                    "working": r.is_success,
                    "latency": round(elapsed, 2),
                    "ip": r.text[:100] if r.is_success else "",
                    "status": r.status_code,
                }
        except Exception as e:
            return {"working": False, "error": str(e)[:100]}

    def needs_premium_proxy(self, last_error: str) -> bool:
        """Determine if we need a premium proxy based on the error type."""
        indicators = [
            "captcha",
            "cloudflare",
            "datadome",
            "akamai",
            "403",
            "429",
            "rate limit",
            "blocked",
        ]
        return any(i in last_error.lower() for i in indicators)

    def get_signup_link(self, provider_tag: str = "") -> str:
        """Get the affiliate signup link for a provider. Records the click for revenue tracking."""
        if not provider_tag:
            provider_tag = "brightdata"
        provider = next((p for p in PREMIUM_PROXY_PROVIDERS if p["tag"] == provider_tag), None)
        if not provider:
            provider = PREMIUM_PROXY_PROVIDERS[0]
        try:
            from referrals import ReferralTracker

            tracker = ReferralTracker()
            tracker.record_click(
                provider["tag"],
                provider["signup_url"],
                source="proxy_manager",
                user_id=os.getenv("USER", ""),
            )
        except Exception as e:
            logger.debug("referral_track_failed", extra={"error": str(e)[:80]})
        return provider["signup_url"]

    def list_providers(self, free: bool = True, premium: bool = True) -> dict[str, Any]:
        """List all available proxy providers."""
        result: dict[str, Any] = {"free": [], "premium": []}
        if free:
            result["free"] = FREE_PROXY_SOURCES
        if premium:
            result["premium"] = PREMIUM_PROXY_PROVIDERS
        return result

    def select_provider(self, provider_tag: str, credentials: dict | None = None) -> dict[str, Any]:
        """Select a premium provider. If credentials provided, save them."""
        provider = next((p for p in PREMIUM_PROXY_PROVIDERS if p["tag"] == provider_tag), None)
        if not provider:
            return {"success": False, "error": f"Unknown provider: {provider_tag}"}

        self.get_signup_link(provider_tag)

        if credentials:
            self.credentials[provider_tag] = ProxyConfig(
                provider=provider_tag,
                proxy_url=credentials.get("proxy_url", f"gate.{provider_tag}.com:8000"),
                username=credentials.get("username", ""),
                password=credentials.get("password", ""),
                api_key=credentials.get("api_key", ""),
                proxy_type=credentials.get("proxy_type", "http"),
            )
            self.active_config = self.credentials[provider_tag]
            self._save_credentials()
            self._save_config()
            return {
                "success": True,
                "provider": provider_tag,
                "config": self.active_config.__dict__,
            }

        return {
            "success": False,
            "needs_signup": True,
            "signup_url": provider["signup_url"],
            "message": (
                f"Visit {provider['signup_url']} to sign up for {provider['name']}, "
                "then call this endpoint again with credentials."
            ),
            "provider": provider,
        }

    def auto_configure_from_env(self) -> bool:
        """Check environment variables for proxy credentials and auto-configure."""
        for provider in PREMIUM_PROXY_PROVIDERS:
            tag = provider["tag"]
            env_prefix = f"PRY_PROXY_{tag.upper()}_"
            url = os.getenv(env_prefix + "URL")
            user = os.getenv(env_prefix + "USERNAME")
            pwd = os.getenv(env_prefix + "PASSWORD")
            api_key = os.getenv(env_prefix + "API_KEY")
            if url or (user and pwd) or api_key:
                self.credentials[tag] = ProxyConfig(
                    provider=tag,
                    proxy_url=url or f"gate.{tag}.com:8000",
                    username=user or "",
                    password=pwd or "",
                    api_key=api_key or "",
                )
                if not self.active_config.provider or self.active_config.provider == "free":
                    self.active_config = self.credentials[tag]
                self._save_credentials()
                self._save_config()
                return True
        return False

    def get_recommendation(self, last_error: str = "") -> dict[str, Any]:
        """Get a recommendation for which proxy provider to use.

        Returns: signup URL, estimated cost, why recommended.
        """
        if not self.needs_premium_proxy(last_error):
            return {"needs_premium": False, "message": "Free fallbacks sufficient"}
        provider = PREMIUM_PROXY_PROVIDERS[0]
        return {
            "needs_premium": True,
            "reason": f"Site appears blocked: '{last_error[:80]}'",
            "recommended_provider": provider["name"],
            "signup_url": self.get_signup_link(provider["tag"]),
            "estimated_cost": provider["min_price"],
            "free_trial": provider.get("free_trial", "Unknown"),
            "all_providers": PREMIUM_PROXY_PROVIDERS[:3],
        }

    def record_referral_click(self, provider_tag: str, user_id: str = "") -> str:
        """Record a referral click for a specific provider. Returns the click_id."""
        provider = next((p for p in PREMIUM_PROXY_PROVIDERS if p["tag"] == provider_tag), None)
        if not provider:
            return ""
        try:
            from referrals import ReferralTracker

            tracker = ReferralTracker()
            return tracker.record_click(
                provider["tag"],
                provider["signup_url"],
                source="proxy_signup",
                user_id=user_id,
            )
        except Exception as e:
            logger.debug("referral_record_failed", extra={"error": str(e)[:80]})
        return ""

    def get_recent_clicks(self, days_back: int = 30) -> list[dict[str, Any]]:
        """Get recent proxy referral clicks for revenue tracking."""
        try:
            from referrals import ReferralTracker

            tracker = ReferralTracker()
            cutoff = datetime.now(UTC).timestamp() - (days_back * 86400)
            return [
                c
                for c in tracker.clicks
                if c.get("source", "").startswith("proxy")
                and _parse_ts(c.get("timestamp", "")) >= cutoff
            ]
        except Exception as e:
            logger.debug("recent_clicks_failed", extra={"error": str(e)[:80]})
            return []


def _parse_ts(ts: str) -> float:
    """Parse an ISO timestamp to epoch seconds. Returns 0 on error."""
    if not ts:
        return 0.0
    try:
        return datetime.fromisoformat(ts).timestamp()
    except (ValueError, TypeError):
        return 0.0