RugMunchMedia/pryscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
pryscraper/referrals.py
cryptorugmunch 232dc16924
All checks were successful
CI / lint (pull_request) Successful in 59s
Details
CI / typecheck (pull_request) Successful in 7m34s
Details
CI / Secret scan (gitleaks) (pull_request) Successful in 1m59s
Details
CI / test (pull_request) Successful in 8m59s
Details
CI / Security audit (bandit) (pull_request) Successful in 1m3s
Details
feat(sql): migrate intelligence, monitor, and referrals to SQLAlchemy 
- intelligence.py: IntelSnapshot table, JSONL replaced with queries.
- monitor.py: Monitor + MonitorRun tables, JSON files replaced.
- structure_monitor.py: StructureSnapshot table for selector history.
- referrals.py: ReferralClick + ReferralConversion + ReferralInAppUsage tables.
- db.py: extended models and import_json_stores for the three domains.
- Added/updated tests: 475 passing.
2026-07-03 01:25:14 +02:00

737 lines
24 KiB
Python
Raw Blame History

"""Pry — Referral / Affiliate revenue system.
Tracks clicks, conversions, and revenue across 60+ providers.
Supports x402 (HTTP 402) pay-per-scrape protocol for monetization."""
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 Rug Munch Media LLC
#
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
# Licensed under MIT. See LICENSE.
import logging
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import func
from sqlalchemy.exc import SQLAlchemyError
from db import ReferralClick, ReferralConversion, ReferralInAppUsage, session_scope
logger = logging.getLogger(__name__)
# Provider catalog with referral program details
# Format: { category: [ {name, url, commission, cookie_days, ...} ] }
PROVIDER_CATALOG: dict[str, list[dict[str, Any]]] = {
"llm": [
{
"name": "OpenAI",
"url": "https://platform.openai.com/signup?via=pry",
"commission": "20% first 6 months",
"cookie_days": 30,
"tag": "openai",
},
{
"name": "Anthropic",
"url": "https://console.anthropic.com/?ref=pry",
"commission": "Enterprise referrals",
"cookie_days": 30,
"tag": "anthropic",
},
{
"name": "Google AI Studio",
"url": "https://aistudio.google.com/?utm_source=pry",
"commission": "Google Cloud Partner tiered",
"cookie_days": 30,
"tag": "google",
},
{
"name": "Cohere",
"url": "https://dashboard.cohere.com/welcome?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "cohere",
},
{
"name": "Mistral AI",
"url": "https://console.mistral.ai/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "mistral",
},
{
"name": "OpenRouter",
"url": "https://openrouter.ai/?ref=pry",
"commission": "$1/signup + usage share",
"cookie_days": 30,
"tag": "openrouter",
},
{
"name": "Groq",
"url": "https://console.groq.com/?ref=pry",
"commission": "Developer program",
"cookie_days": 30,
"tag": "groq",
},
{
"name": "Together AI",
"url": "https://api.together.xyz/?ref=pry",
"commission": "$5/signup + usage share",
"cookie_days": 30,
"tag": "together",
},
{
"name": "Replicate",
"url": "https://replicate.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "replicate",
},
{
"name": "Fireworks AI",
"url": "https://fireworks.ai/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "fireworks",
},
{
"name": "xAI (Grok)",
"url": "https://console.x.ai/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "xai",
},
{
"name": "DeepInfra",
"url": "https://deepinfra.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "deepinfra",
},
{
"name": "Novita AI",
"url": "https://novita.ai/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "novita",
},
{
"name": "Anyscale",
"url": "https://anyscale.com/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "anyscale",
},
{
"name": "Lepton AI",
"url": "https://www.lepton.ai/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "lepton",
},
],
"hosting": [
{
"name": "Hetzner",
"url": "https://hetzner.com/?ref=pry",
"commission": "\u20ac25/signup + 10% recurring",
"cookie_days": 90,
"tag": "hetzner",
},
{
"name": "DigitalOcean",
"url": "https://m.do.co/c/pry",
"commission": "$25/paid referral",
"cookie_days": 60,
"tag": "do",
},
{
"name": "Linode (Akamai)",
"url": "https://www.linode.com/?ref=pry",
"commission": "$20/paid referral",
"cookie_days": 60,
"tag": "linode",
},
{
"name": "Vultr",
"url": "https://www.vultr.com/?ref=pry",
"commission": "$10-$50/signup",
"cookie_days": 30,
"tag": "vultr",
},
{
"name": "OVHcloud",
"url": "https://www.ovhcloud.com/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "ovh",
},
{
"name": "UpCloud",
"url": "https://upcloud.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "upcloud",
},
{
"name": "Netcup",
"url": "https://netcup.com/?ref=pry",
"commission": "\u20ac5/signup",
"cookie_days": 90,
"tag": "netcup",
},
],
"domains": [
{
"name": "Namecheap",
"url": "https://namecheap.com/?affiliateid=pry",
"commission": "Up to $50/domain",
"cookie_days": 365,
"tag": "namecheap",
},
{
"name": "Cloudflare Registrar",
"url": "https://cloudflare.com/?ref=pry",
"commission": "At-cost pricing",
"cookie_days": 0,
"tag": "cloudflare",
},
{
"name": "Porkbun",
"url": "https://porkbun.com/?affiliate=pry",
"commission": "$0.50-$2/domain",
"cookie_days": 365,
"tag": "porkbun",
},
{
"name": "Spaceship",
"url": "https://spaceship.com/?affiliate=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "spaceship",
},
],
"cdn": [
{
"name": "Cloudflare",
"url": "https://cloudflare.com/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "cloudflare",
},
{
"name": "Bunny CDN",
"url": "https://bunny.net?affiliate=pry",
"commission": "10% recurring lifetime",
"cookie_days": 60,
"tag": "bunny",
},
{
"name": "Fastly",
"url": "https://fastly.com/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "fastly",
},
],
"email": [
{
"name": "SendGrid",
"url": "https://sendgrid.com/?ref=pry",
"commission": "$30/referral",
"cookie_days": 30,
"tag": "sendgrid",
},
{
"name": "Mailgun",
"url": "https://mailgun.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "mailgun",
},
{
"name": "Resend",
"url": "https://resend.com/?ref=pry",
"commission": "$20/signup",
"cookie_days": 30,
"tag": "resend",
},
{
"name": "Postmark",
"url": "https://postmarkapp.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "postmark",
},
{
"name": "Brevo (Sendinblue)",
"url": "https://www.brevo.com/?ref=pry",
"commission": "Paid plan share",
"cookie_days": 30,
"tag": "brevo",
},
],
"monitoring": [
{
"name": "Sentry",
"url": "https://sentry.io/?ref=pry",
"commission": "$100-$1,000 (tiered)",
"cookie_days": 60,
"tag": "sentry",
},
{
"name": "Datadog",
"url": "https://datadoghq.com/?ref=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "datadog",
},
{
"name": "Better Stack",
"url": "https://betterstack.com/?ref=pry",
"commission": "$50/paid signup",
"cookie_days": 30,
"tag": "betterstack",
},
{
"name": "Highlight.io",
"url": "https://highlight.io/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "highlight",
},
{
"name": "PostHog",
"url": "https://posthog.com/?ref=pry",
"commission": "20% recurring for 2 years",
"cookie_days": 60,
"tag": "posthog",
},
],
"proxies": [
{
"name": "Bright Data",
"url": "https://brightdata.com/cp/start?aff_id=pry",
"commission": "$2-$50/signup, up to 10% recurring",
"cookie_days": 30,
"tag": "brightdata",
"free_trial": "$5 credit",
"min_price": "$0.10/GB residential",
"note": "Industry leader. Best for high-volume scraping.",
},
{
"name": "Smartproxy",
"url": "https://dashboard.smartproxy.com/registration?aff_id=pry",
"commission": "20% recurring lifetime",
"cookie_days": 60,
"tag": "smartproxy",
"free_trial": "100MB free",
"min_price": "$0.10/GB residential",
"note": "Good balance of price and quality.",
},
{
"name": "Oxylabs",
"url": "https://dashboard.oxylabs.io/?aff_id=pry",
"commission": "10-30% recurring",
"cookie_days": 30,
"tag": "oxylabs",
"free_trial": "5K requests",
"min_price": "$1.20/GB residential",
"note": "Enterprise-grade. Best for big data scraping.",
},
{
"name": "IPRoyal",
"url": "https://dashboard.iproyal.com/signup?aff=pry",
"commission": "30% recurring lifetime",
"cookie_days": 60,
"tag": "iproyal",
"free_trial": "None",
"min_price": "$0.04/GB residential",
"note": "Cheapest. Good for budget scraping.",
},
{
"name": "Webshare",
"url": "https://proxy.webshare.io/register?aff=pry",
"commission": "30% recurring lifetime",
"cookie_days": 60,
"tag": "webshare",
"free_trial": "10 free proxies",
"min_price": "$0.03/proxy/month",
"note": "Best free tier. Try before you buy.",
},
{
"name": "Proxy-Seller",
"url": "https://proxy-seller.com/?partner=pry",
"commission": "30% recurring",
"cookie_days": 60,
"tag": "proxyseller",
"free_trial": "None",
"min_price": "$1.40/proxy/month",
"note": "Cheap private proxies.",
},
{
"name": "NetNut",
"url": "https://netnut.io/?aff=pry",
"commission": "Partner program",
"cookie_days": 30,
"tag": "netnut",
"free_trial": "7 days free",
"min_price": "$0.20/GB",
"note": "Fast residential IPs from ISPs.",
},
{
"name": "ProxyMesh",
"url": "https://proxymesh.com/?aff=pry",
"commission": "20% recurring",
"cookie_days": 30,
"tag": "proxymesh",
"free_trial": "None",
"min_price": "$0.80/proxy",
"note": "Simple rotating proxies.",
},
{
"name": "Decodo (Smartproxy)",
"url": "https://decodo.com/signup?aff=pry",
"commission": "20% recurring",
"cookie_days": 60,
"tag": "decodo",
"note": "Budget option from Smartproxy.",
},
{
"name": "PacketStream",
"url": "https://packetstream.io/signup?aff=pry",
"commission": "20% revenue share",
"cookie_days": 60,
"tag": "packetstream",
"free_trial": "Pay-as-you-go",
"min_price": "$0.05/GB",
"note": "Bandwidth-sharing residential proxies.",
},
],
"voice": [
{
"name": "ElevenLabs",
"url": "https://elevenlabs.io/?ref=pry",
"commission": "22% recurring for 12 months",
"cookie_days": 30,
"tag": "elevenlabs",
},
{
"name": "Murf AI",
"url": "https://murf.ai/?ref=pry",
"commission": "30% recurring",
"cookie_days": 30,
"tag": "murf",
},
{
"name": "Play.ht",
"url": "https://play.ht/?ref=pry",
"commission": "20% recurring",
"cookie_days": 30,
"tag": "playht",
},
],
"media": [
{
"name": "RunwayML",
"url": "https://runwayml.com/?ref=pry",
"commission": "20% first year",
"cookie_days": 30,
"tag": "runway",
},
{
"name": "HeyGen",
"url": "https://heygen.com/?ref=pry",
"commission": "20% recurring for 12 months",
"cookie_days": 30,
"tag": "heygen",
},
{
"name": "Synthesia",
"url": "https://synthesia.io/?ref=pry",
"commission": "20% recurring",
"cookie_days": 30,
"tag": "synthesia",
},
{
"name": "Pika",
"url": "https://pika.art/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "pika",
},
{
"name": "Suno",
"url": "https://suno.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "suno",
},
{
"name": "ElevenLabs Image",
"url": "https://elevenlabs.io/image?ref=pry",
"commission": "22% recurring",
"cookie_days": 30,
"tag": "elevenlabs-image",
},
],
"devtools": [
{
"name": "Cursor",
"url": "https://cursor.com/?ref=pry",
"commission": "Varies",
"cookie_days": 30,
"tag": "cursor",
},
{
"name": "v0.dev",
"url": "https://v0.dev/?ref=pry",
"commission": "Varies",
"cookie_days": 30,
"tag": "v0",
},
{
"name": "Bolt.new",
"url": "https://bolt.new/?ref=pry",
"commission": "Varies",
"cookie_days": 30,
"tag": "bolt",
},
{
"name": "Lovable",
"url": "https://lovable.dev/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "lovable",
},
{
"name": "Replit",
"url": "https://replit.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "replit",
},
],
"search": [
{
"name": "Algolia",
"url": "https://algolia.com/?ref=pry",
"commission": "$200/paid signup",
"cookie_days": 30,
"tag": "algolia",
},
{
"name": "Meilisearch Cloud",
"url": "https://meilisearch.com/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "meilisearch",
},
{
"name": "Typesense Cloud",
"url": "https://typesense.org/?ref=pry",
"commission": "Usage share",
"cookie_days": 30,
"tag": "typesense",
},
],
"captcha": [
{
"name": "Capsolver",
"url": "https://capsolver.com/?ref=pry",
"commission": "20% recurring lifetime",
"cookie_days": 60,
"tag": "capsolver",
},
{
"name": "2Captcha",
"url": "https://2captcha.com/?ref=pry",
"commission": "10% recurring lifetime",
"cookie_days": 60,
"tag": "2captcha",
},
{
"name": "Anti-Captcha",
"url": "https://anti-captcha.com/?ref=pry",
"commission": "10% recurring lifetime",
"cookie_days": 60,
"tag": "anticaptcha",
},
{
"name": "CapMonster Cloud",
"url": "https://capmonster.cloud/?ref=pry",
"commission": "10% recurring",
"cookie_days": 60,
"tag": "capmonster",
},
{
"name": "NextCaptcha",
"url": "https://nextcaptcha.com/?ref=pry",
"commission": "10% recurring",
"cookie_days": 60,
"tag": "nextcaptcha",
},
],
"flaresolverr": [
{
"name": "FlareSolverr (Self-hosted)",
"url": "https://github.com/FlareSolverr/FlareSolverr",
"commission": "Open source (no affiliate)",
"cookie_days": 0,
"tag": "flaresolverr",
"note": "Self-host for free; recommend on Hydra",
},
],
}
class ReferralTracker:
"""Track referral clicks, conversions, and revenue."""
def __init__(self) -> None:
pass
def record_click(
self, provider_tag: str, url: str, source: str = "pry", user_id: str = ""
) -> str:
"""Record a referral link click. Returns the tracking ID."""
click_id = uuid.uuid4().hex[:12]
try:
with session_scope() as s:
s.add(
ReferralClick(
click_id=click_id,
provider=provider_tag,
url=url,
source=source,
user_id=user_id,
converted=False,
conversion_value_usd=0.0,
)
)
except SQLAlchemyError as e:
logger.error(
"referral_click_failed",
extra={"click_id": click_id, "provider": provider_tag, "error": str(e)[:200]},
)
raise
logger.info("referral_click", extra={"provider": provider_tag, "source": source})
return click_id
def record_conversion(self, click_id: str, revenue_usd: float = 0.0, notes: str = "") -> bool:
"""Record a conversion for a click. Returns True if successful."""
try:
with session_scope() as s:
click = s.query(ReferralClick).filter_by(click_id=click_id).first()
if click is None:
logger.warning(
"referral_conversion_click_not_found",
extra={"click_id": click_id},
)
return False
click.converted = True
click.conversion_value_usd = revenue_usd
s.add(
ReferralConversion(
click_id=click_id,
provider=click.provider,
revenue_usd=revenue_usd,
notes=notes,
)
)
except SQLAlchemyError as e:
logger.error(
"referral_conversion_failed",
extra={"click_id": click_id, "error": str(e)[:200]},
)
raise
logger.info(
"referral_conversion",
extra={"click_id": click_id, "revenue_usd": revenue_usd},
)
return True
def get_catalog(self, category: str = "") -> list[dict[str, Any]]:
"""Get the provider catalog, optionally filtered by category."""
if category:
return PROVIDER_CATALOG.get(category, [])
return [
{"category": cat, "providers": providers} for cat, providers in PROVIDER_CATALOG.items()
]
def get_stats(self, days_back: int = 30) -> dict[str, Any]:
"""Get referral statistics for the last N days."""
cutoff = datetime.now(UTC) - timedelta(days=days_back)
try:
with session_scope() as s:
total_clicks = (
s.query(func.count(ReferralClick.id))
.filter(ReferralClick.ts >= cutoff)
.scalar()
or 0
)
click_rows = (
s.query(
ReferralClick.provider,
func.count(ReferralClick.id).label("cnt"),
)
.filter(ReferralClick.ts >= cutoff)
.group_by(ReferralClick.provider)
.all()
)
clicks_by_provider = {r.provider: r.cnt for r in click_rows}
conv_rows = (
s.query(
ReferralConversion.provider,
func.count(ReferralConversion.id).label("cnt"),
func.sum(ReferralConversion.revenue_usd).label("rev"),
)
.filter(ReferralConversion.ts >= cutoff)
.group_by(ReferralConversion.provider)
.all()
)
conversions_by_provider = {r.provider: r.cnt for r in conv_rows}
revenue_by_provider = {r.provider: float(r.rev or 0.0) for r in conv_rows}
total_conversions = sum(conversions_by_provider.values())
total_revenue = sum(revenue_by_provider.values())
return {
"period_days": days_back,
"total_clicks": total_clicks,
"total_conversions": total_conversions,
"clicks_by_provider": clicks_by_provider,
"conversions_by_provider": conversions_by_provider,
"revenue_by_provider_usd": revenue_by_provider,
"total_revenue_usd": total_revenue,
"conversion_rate": total_conversions / max(total_clicks, 1),
"catalog_size": sum(len(v) for v in PROVIDER_CATALOG.values()),
}
except SQLAlchemyError as e:
logger.error(
"referral_stats_failed",
extra={"error": str(e)[:200]},
)
raise
def track_in_app_usage(self, provider_tag: str, revenue_usd: float) -> None:
"""Track revenue from in-app LLM provider usage (per-call)."""
try:
with session_scope() as s:
s.add(
ReferralInAppUsage(
provider=provider_tag,
revenue_usd=revenue_usd,
)
)
except SQLAlchemyError as e:
logger.error(
"referral_in_app_usage_failed",
extra={"provider": provider_tag, "error": str(e)[:200]},
)
raise
logger.info(
"referral_in_app_usage",
extra={"provider": provider_tag, "revenue_usd": revenue_usd},
)
Reference in a new issue View git blame Copy permalink