docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
464
proxy_manager.py
Normal file
464
proxy_manager.py
Normal file
|
|
@ -0,0 +1,464 @@
|
|||
"""Pry — Proxy Manager with affiliate-signup flow.
|
||||
Tries free proxies first (Tor, public rotating). When premium proxy is needed,
|
||||
prompts user to sign up via affiliate links. Pre-wired to 5+ providers."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROXY_DIR = Path(os.path.expanduser("~/.pry/proxies"))
|
||||
PROXY_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Free proxy sources (no signup required)
|
||||
FREE_PROXY_SOURCES = [
|
||||
{
|
||||
"name": "Tor",
|
||||
"type": "socks5",
|
||||
"url": "socks5://127.0.0.1:9050",
|
||||
"cost": "Free",
|
||||
"speed": "Slow",
|
||||
"reliability": "High",
|
||||
},
|
||||
{
|
||||
"name": "Public Pool (round-robin)",
|
||||
"type": "http",
|
||||
"url": "rotating",
|
||||
"cost": "Free",
|
||||
"speed": "Variable",
|
||||
"reliability": "Low",
|
||||
},
|
||||
{
|
||||
"name": "Cloudflare WARP",
|
||||
"type": "wireguard",
|
||||
"url": "warp://1.1.1.1",
|
||||
"cost": "Free",
|
||||
"speed": "Fast",
|
||||
"reliability": "High",
|
||||
},
|
||||
]
|
||||
|
||||
# Premium proxy providers with full affiliate details
|
||||
PREMIUM_PROXY_PROVIDERS = [
|
||||
{
|
||||
"name": "Bright Data",
|
||||
"tag": "brightdata",
|
||||
"url": "https://brightdata.com/?ref=pry",
|
||||
"signup_url": "https://brightdata.com/cp/start?aff_id=pry",
|
||||
"api_endpoint": "https://api.brightdata.com",
|
||||
"auth_format": "username:password",
|
||||
"commission": "$2-$50 per signup, up to 10% recurring",
|
||||
"cookie_days": 30,
|
||||
"free_trial": "Yes, $5 credit",
|
||||
"min_price": "$0.10/GB residential",
|
||||
"speed": "Very Fast",
|
||||
"reliability": "Very High",
|
||||
"ips": "72M+ residential, 770K+ datacenter",
|
||||
"geo_coverage": "195 countries",
|
||||
"protocols": ["HTTP", "SOCKS5", "HTTPS"],
|
||||
"note": "Industry leader. Best for high-volume scraping.",
|
||||
},
|
||||
{
|
||||
"name": "Smartproxy",
|
||||
"tag": "smartproxy",
|
||||
"url": "https://smartproxy.com/?ref=pry",
|
||||
"signup_url": "https://dashboard.smartproxy.com/registration?aff_id=pry",
|
||||
"api_endpoint": "https://api.smartproxy.com",
|
||||
"auth_format": "username:password",
|
||||
"commission": "20% recurring lifetime",
|
||||
"cookie_days": 60,
|
||||
"free_trial": "Yes, 100MB",
|
||||
"min_price": "$0.10/GB residential",
|
||||
"speed": "Fast",
|
||||
"reliability": "High",
|
||||
"ips": "40M+ residential",
|
||||
"geo_coverage": "195 countries",
|
||||
"protocols": ["HTTP", "SOCKS5"],
|
||||
"note": "Good balance of price and quality.",
|
||||
},
|
||||
{
|
||||
"name": "Oxylabs",
|
||||
"tag": "oxylabs",
|
||||
"url": "https://oxylabs.io/?ref=pry",
|
||||
"signup_url": "https://dashboard.oxylabs.io/?aff_id=pry",
|
||||
"api_endpoint": "https://api.oxylabs.com",
|
||||
"auth_format": "username:password",
|
||||
"commission": "Partner program (10-30% recurring)",
|
||||
"cookie_days": 30,
|
||||
"free_trial": "Yes, 5K requests",
|
||||
"min_price": "$1.20/GB residential",
|
||||
"speed": "Very Fast",
|
||||
"reliability": "Very High",
|
||||
"ips": "100M+ residential",
|
||||
"geo_coverage": "195 countries",
|
||||
"protocols": ["HTTP", "SOCKS5", "HTTPS"],
|
||||
"note": "Enterprise-grade. Best for big data scraping.",
|
||||
},
|
||||
{
|
||||
"name": "IPRoyal",
|
||||
"tag": "iproyal",
|
||||
"url": "https://iproyal.com/?ref=pry",
|
||||
"signup_url": "https://dashboard.iproyal.com/signup?aff=pry",
|
||||
"api_endpoint": "https://api.iproyal.com",
|
||||
"auth_format": "username:password",
|
||||
"commission": "30% recurring lifetime",
|
||||
"cookie_days": 60,
|
||||
"free_trial": "No",
|
||||
"min_price": "$0.04/GB residential",
|
||||
"speed": "Medium",
|
||||
"reliability": "High",
|
||||
"ips": "300K+ residential",
|
||||
"geo_coverage": "150+ countries",
|
||||
"protocols": ["HTTP", "SOCKS5"],
|
||||
"note": "Cheapest. Good for budget scraping.",
|
||||
},
|
||||
{
|
||||
"name": "Webshare",
|
||||
"tag": "webshare",
|
||||
"url": "https://www.webshare.io/?ref=pry",
|
||||
"signup_url": "https://proxy.webshare.io/register?aff=pry",
|
||||
"api_endpoint": "https://proxy.webshare.io/api/v2",
|
||||
"auth_format": "api_key",
|
||||
"commission": "30% recurring lifetime",
|
||||
"cookie_days": 60,
|
||||
"free_trial": "Yes, 10 free proxies",
|
||||
"min_price": "$0.03/proxy/month (datacenter)",
|
||||
"speed": "Very Fast",
|
||||
"reliability": "High",
|
||||
"ips": "500K+ datacenter/residential",
|
||||
"geo_coverage": "100+ countries",
|
||||
"protocols": ["HTTP", "SOCKS5"],
|
||||
"note": "Best free tier. Try before you buy.",
|
||||
},
|
||||
{
|
||||
"name": "Proxy-Seller",
|
||||
"tag": "proxyseller",
|
||||
"url": "https://proxy-seller.com/?ref=pry",
|
||||
"signup_url": "https://proxy-seller.com/?partner=pry",
|
||||
"auth_format": "username:password",
|
||||
"commission": "30% recurring",
|
||||
"cookie_days": 60,
|
||||
"free_trial": "No",
|
||||
"min_price": "$1.40/proxy/month",
|
||||
"speed": "Fast",
|
||||
"reliability": "High",
|
||||
"note": "Cheap private proxies.",
|
||||
},
|
||||
{
|
||||
"name": "NetNut",
|
||||
"tag": "netnut",
|
||||
"url": "https://netnut.io/?ref=pry",
|
||||
"signup_url": "https://netnut.io/?aff=pry",
|
||||
"auth_format": "username:password",
|
||||
"commission": "Partner program",
|
||||
"cookie_days": 30,
|
||||
"free_trial": "Yes, 7 days",
|
||||
"min_price": "$0.20/GB",
|
||||
"note": "Fast residential IPs from ISPs.",
|
||||
},
|
||||
{
|
||||
"name": "ProxyMesh",
|
||||
"tag": "proxymesh",
|
||||
"url": "https://proxymesh.com/?ref=pry",
|
||||
"signup_url": "https://proxymesh.com/?aff=pry",
|
||||
"auth_format": "username:password",
|
||||
"commission": "20% recurring",
|
||||
"cookie_days": 30,
|
||||
"free_trial": "No",
|
||||
"min_price": "$0.80/proxy",
|
||||
"note": "Simple rotating proxies.",
|
||||
},
|
||||
{
|
||||
"name": "Decodo (Smartproxy)",
|
||||
"tag": "decodo",
|
||||
"url": "https://decodo.com/?ref=pry",
|
||||
"signup_url": "https://decodo.com/signup?aff=pry",
|
||||
"auth_format": "username:password",
|
||||
"commission": "20% recurring",
|
||||
"cookie_days": 60,
|
||||
"note": "Budget option from Smartproxy.",
|
||||
},
|
||||
{
|
||||
"name": "PacketStream",
|
||||
"tag": "packetstream",
|
||||
"url": "https://packetstream.io/?ref=pry",
|
||||
"signup_url": "https://packetstream.io/signup?aff=pry",
|
||||
"auth_format": "username:password",
|
||||
"commission": "20% revenue share",
|
||||
"cookie_days": 60,
|
||||
"free_trial": "Pay-as-you-go",
|
||||
"min_price": "$0.05/GB",
|
||||
"note": "Bandwidth-sharing residential proxies.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyConfig:
|
||||
"""Proxy configuration."""
|
||||
|
||||
provider: str = "free"
|
||||
proxy_url: str = ""
|
||||
username: str = ""
|
||||
password: str = ""
|
||||
api_key: str = ""
|
||||
proxy_type: str = "http"
|
||||
geo: str = "auto"
|
||||
rotation: str = "per_request"
|
||||
auto_rotate: bool = True
|
||||
|
||||
|
||||
class ProxyManager:
|
||||
"""Manages proxy selection, free fallbacks, and premium signup flow."""
|
||||
|
||||
def __init__(self, data_dir: Path | None = None) -> None:
|
||||
self.data_dir = data_dir or PROXY_DIR
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.config_file = self.data_dir / "active_config.json"
|
||||
self.creds_file = self.data_dir / "credentials.json"
|
||||
self.active_config = self._load_config()
|
||||
self.credentials: dict[str, ProxyConfig] = self._load_credentials()
|
||||
self.auto_configure_from_env()
|
||||
|
||||
def _load_config(self) -> ProxyConfig:
|
||||
if self.config_file.exists():
|
||||
try:
|
||||
return ProxyConfig(**json.loads(self.config_file.read_text()))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return ProxyConfig()
|
||||
|
||||
def _load_credentials(self) -> dict[str, ProxyConfig]:
|
||||
if self.creds_file.exists():
|
||||
try:
|
||||
data = json.loads(self.creds_file.read_text())
|
||||
return {k: ProxyConfig(**v) for k, v in data.items()}
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _save_config(self) -> None:
|
||||
try:
|
||||
self.config_file.write_text(json.dumps(self.active_config.__dict__, indent=2))
|
||||
except OSError as e:
|
||||
logger.debug("proxy_config_save_failed", extra={"error": str(e)[:80]})
|
||||
|
||||
def _save_credentials(self) -> None:
|
||||
try:
|
||||
data = {k: v.__dict__ for k, v in self.credentials.items()}
|
||||
self.creds_file.write_text(json.dumps(data, indent=2, default=str))
|
||||
except OSError as e:
|
||||
logger.debug("proxy_creds_save_failed", extra={"error": str(e)[:80]})
|
||||
|
||||
def get_proxy_url(self) -> str | None:
|
||||
"""Get the active proxy URL. Returns None if no proxy configured."""
|
||||
c = self.active_config
|
||||
if c.provider == "free":
|
||||
if c.proxy_url and c.proxy_url != "rotating":
|
||||
return c.proxy_url
|
||||
return None
|
||||
if c.provider not in self.credentials:
|
||||
return None
|
||||
cred = self.credentials[c.provider]
|
||||
if cred.api_key:
|
||||
return f"http://{cred.api_key}@proxy.{cred.provider}.com:8000"
|
||||
if cred.username and cred.password:
|
||||
return f"{cred.proxy_type}://{cred.username}:{cred.password}@{cred.proxy_url}"
|
||||
if cred.proxy_url:
|
||||
return cred.proxy_url
|
||||
return None
|
||||
|
||||
def test_proxy(
|
||||
self, proxy_url: str, test_url: str = "https://httpbin.org/ip", timeout: int = 10
|
||||
) -> dict[str, Any]:
|
||||
"""Test a proxy and return its public IP, latency, and working status."""
|
||||
import httpx
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
with httpx.Client(proxy=proxy_url, timeout=timeout) as c:
|
||||
r = c.get(test_url)
|
||||
elapsed = time.time() - start
|
||||
return {
|
||||
"working": r.is_success,
|
||||
"latency": round(elapsed, 2),
|
||||
"ip": r.text[:100] if r.is_success else "",
|
||||
"status": r.status_code,
|
||||
}
|
||||
except Exception as e:
|
||||
return {"working": False, "error": str(e)[:100]}
|
||||
|
||||
def needs_premium_proxy(self, last_error: str) -> bool:
|
||||
"""Determine if we need a premium proxy based on the error type."""
|
||||
indicators = [
|
||||
"captcha",
|
||||
"cloudflare",
|
||||
"datadome",
|
||||
"akamai",
|
||||
"403",
|
||||
"429",
|
||||
"rate limit",
|
||||
"blocked",
|
||||
]
|
||||
return any(i in last_error.lower() for i in indicators)
|
||||
|
||||
def get_signup_link(self, provider_tag: str = "") -> str:
|
||||
"""Get the affiliate signup link for a provider. Records the click for revenue tracking."""
|
||||
if not provider_tag:
|
||||
provider_tag = "brightdata"
|
||||
provider = next((p for p in PREMIUM_PROXY_PROVIDERS if p["tag"] == provider_tag), None)
|
||||
if not provider:
|
||||
provider = PREMIUM_PROXY_PROVIDERS[0]
|
||||
try:
|
||||
from referrals import ReferralTracker
|
||||
|
||||
tracker = ReferralTracker()
|
||||
tracker.record_click(
|
||||
provider["tag"],
|
||||
provider["signup_url"],
|
||||
source="proxy_manager",
|
||||
user_id=os.getenv("USER", ""),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("referral_track_failed", extra={"error": str(e)[:80]})
|
||||
return provider["signup_url"]
|
||||
|
||||
def list_providers(self, free: bool = True, premium: bool = True) -> dict[str, Any]:
|
||||
"""List all available proxy providers."""
|
||||
result: dict[str, Any] = {"free": [], "premium": []}
|
||||
if free:
|
||||
result["free"] = FREE_PROXY_SOURCES
|
||||
if premium:
|
||||
result["premium"] = PREMIUM_PROXY_PROVIDERS
|
||||
return result
|
||||
|
||||
def select_provider(self, provider_tag: str, credentials: dict | None = None) -> dict[str, Any]:
|
||||
"""Select a premium provider. If credentials provided, save them."""
|
||||
provider = next((p for p in PREMIUM_PROXY_PROVIDERS if p["tag"] == provider_tag), None)
|
||||
if not provider:
|
||||
return {"success": False, "error": f"Unknown provider: {provider_tag}"}
|
||||
|
||||
self.get_signup_link(provider_tag)
|
||||
|
||||
if credentials:
|
||||
self.credentials[provider_tag] = ProxyConfig(
|
||||
provider=provider_tag,
|
||||
proxy_url=credentials.get("proxy_url", f"gate.{provider_tag}.com:8000"),
|
||||
username=credentials.get("username", ""),
|
||||
password=credentials.get("password", ""),
|
||||
api_key=credentials.get("api_key", ""),
|
||||
proxy_type=credentials.get("proxy_type", "http"),
|
||||
)
|
||||
self.active_config = self.credentials[provider_tag]
|
||||
self._save_credentials()
|
||||
self._save_config()
|
||||
return {
|
||||
"success": True,
|
||||
"provider": provider_tag,
|
||||
"config": self.active_config.__dict__,
|
||||
}
|
||||
|
||||
return {
|
||||
"success": False,
|
||||
"needs_signup": True,
|
||||
"signup_url": provider["signup_url"],
|
||||
"message": (
|
||||
f"Visit {provider['signup_url']} to sign up for {provider['name']}, "
|
||||
"then call this endpoint again with credentials."
|
||||
),
|
||||
"provider": provider,
|
||||
}
|
||||
|
||||
def auto_configure_from_env(self) -> bool:
|
||||
"""Check environment variables for proxy credentials and auto-configure."""
|
||||
for provider in PREMIUM_PROXY_PROVIDERS:
|
||||
tag = provider["tag"]
|
||||
env_prefix = f"PRY_PROXY_{tag.upper()}_"
|
||||
url = os.getenv(env_prefix + "URL")
|
||||
user = os.getenv(env_prefix + "USERNAME")
|
||||
pwd = os.getenv(env_prefix + "PASSWORD")
|
||||
api_key = os.getenv(env_prefix + "API_KEY")
|
||||
if url or (user and pwd) or api_key:
|
||||
self.credentials[tag] = ProxyConfig(
|
||||
provider=tag,
|
||||
proxy_url=url or f"gate.{tag}.com:8000",
|
||||
username=user or "",
|
||||
password=pwd or "",
|
||||
api_key=api_key or "",
|
||||
)
|
||||
if not self.active_config.provider or self.active_config.provider == "free":
|
||||
self.active_config = self.credentials[tag]
|
||||
self._save_credentials()
|
||||
self._save_config()
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_recommendation(self, last_error: str = "") -> dict[str, Any]:
|
||||
"""Get a recommendation for which proxy provider to use.
|
||||
|
||||
Returns: signup URL, estimated cost, why recommended.
|
||||
"""
|
||||
if not self.needs_premium_proxy(last_error):
|
||||
return {"needs_premium": False, "message": "Free fallbacks sufficient"}
|
||||
provider = PREMIUM_PROXY_PROVIDERS[0]
|
||||
return {
|
||||
"needs_premium": True,
|
||||
"reason": f"Site appears blocked: '{last_error[:80]}'",
|
||||
"recommended_provider": provider["name"],
|
||||
"signup_url": self.get_signup_link(provider["tag"]),
|
||||
"estimated_cost": provider["min_price"],
|
||||
"free_trial": provider.get("free_trial", "Unknown"),
|
||||
"all_providers": PREMIUM_PROXY_PROVIDERS[:3],
|
||||
}
|
||||
|
||||
def record_referral_click(self, provider_tag: str, user_id: str = "") -> str:
|
||||
"""Record a referral click for a specific provider. Returns the click_id."""
|
||||
provider = next((p for p in PREMIUM_PROXY_PROVIDERS if p["tag"] == provider_tag), None)
|
||||
if not provider:
|
||||
return ""
|
||||
try:
|
||||
from referrals import ReferralTracker
|
||||
|
||||
tracker = ReferralTracker()
|
||||
return tracker.record_click(
|
||||
provider["tag"],
|
||||
provider["signup_url"],
|
||||
source="proxy_signup",
|
||||
user_id=user_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("referral_record_failed", extra={"error": str(e)[:80]})
|
||||
return ""
|
||||
|
||||
def get_recent_clicks(self, days_back: int = 30) -> list[dict[str, Any]]:
|
||||
"""Get recent proxy referral clicks for revenue tracking."""
|
||||
try:
|
||||
from referrals import ReferralTracker
|
||||
|
||||
tracker = ReferralTracker()
|
||||
cutoff = datetime.now(UTC).timestamp() - (days_back * 86400)
|
||||
return [
|
||||
c
|
||||
for c in tracker.clicks
|
||||
if c.get("source", "").startswith("proxy")
|
||||
and _parse_ts(c.get("timestamp", "")) >= cutoff
|
||||
]
|
||||
except Exception as e:
|
||||
logger.debug("recent_clicks_failed", extra={"error": str(e)[:80]})
|
||||
return []
|
||||
|
||||
|
||||
def _parse_ts(ts: str) -> float:
|
||||
"""Parse an ISO timestamp to epoch seconds. Returns 0 on error."""
|
||||
if not ts:
|
||||
return 0.0
|
||||
try:
|
||||
return datetime.fromisoformat(ts).timestamp()
|
||||
except (ValueError, TypeError):
|
||||
return 0.0
|
||||
Loading…
Add table
Add a link
Reference in a new issue