bb77eb5f35
Re-license Pry from full Proprietary to a dual-license model: - Core engine, extraction, templates (80+), MCP server, x402 payment rail, CLI, SDK, browser extension, WordPress plugin, Shopify app, and llm_providers: MIT (see LICENSE) - Anti-detection / stealth subset (15 files): BSL 1.1 with Change Date 2029-01-01 (see LICENSE-BSL-STEALTH) BSL files (anti-detection moat): ultimate_scraper.py, stealth_engine.py, stealth_scripts/*.js (6), camoufox_integration.py, tls_fingerprint.py, cookie_warmer.py, behavioral_biometrics.py, adaptive.py, browser_pool.py, network.py, captcha_solver.py, shadow_dom.py, lazy_load.py, signup_automator.py, auth_connector.py This enables community contributions to the core engine (templates, integrations, MCP tools) while protecting the anti-detection techniques that constitute the actual competitive moat. BSL Additional Use Grant permits free non-production use; production deployment requires a commercial license from enterprise@rugmunch.io. Changes: - Replace proprietary LICENSE with MIT LICENSE + new LICENSE-BSL-STEALTH - Add SPDX-License-Identifier headers to 300+ source files - Add docs/adr/0002-dual-licensing.md (ADR documenting the decision) - Update README.md: new License section with BSL Additional Use Grant - Update LICENSING_PRICING_STRATEGY.md: Section 3 (PryScraper) for dual license - Update AGENTS.md: license line in header + new rule 8 (PRs touching BSL rejected) - Update pyproject.toml: license = "MIT AND BSL-1.1" + classifiers + license-files - Update DECISIONS.md index with ADR-0002 - Update STATUS.md (2026-07-03) and PLAN.md sprint goals Refs: ADR-0002
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Pry — token-bucket rate limiter per IP.
|
|
Prevents abuse while keeping the service responsive."""
|
|
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
|
|
import time
|
|
|
|
|
|
class RateLimiter:
|
|
"""Token-bucket rate limiter with per-IP tracking.
|
|
|
|
Default: 60 requests/minute per IP, burstable to 100.
|
|
"""
|
|
|
|
def __init__(self, default_rpm: int = 60, burst: int = 100):
|
|
self.default_rpm = default_rpm
|
|
self.burst = burst
|
|
self._buckets: dict[str, dict] = {}
|
|
self._cleanup_interval = 300 # 5 min cleanup
|
|
self._last_cleanup = time.time()
|
|
|
|
def check(self, ip: str, rpm: int | None = None) -> tuple[bool, dict]:
|
|
"""Check if request is allowed. Returns (allowed, stats)."""
|
|
now = time.time()
|
|
rate = rpm or self.default_rpm
|
|
|
|
# Periodic cleanup
|
|
if now - self._last_cleanup > self._cleanup_interval:
|
|
self._cleanup()
|
|
|
|
# Get or create bucket
|
|
if ip not in self._buckets:
|
|
self._buckets[ip] = {
|
|
"tokens": self.burst,
|
|
"last_refill": now,
|
|
"total": 0,
|
|
"blocked": 0,
|
|
}
|
|
|
|
bucket = self._buckets[ip]
|
|
|
|
# Refill tokens
|
|
elapsed = now - bucket["last_refill"]
|
|
bucket["tokens"] = min(self.burst, bucket["tokens"] + elapsed * (rate / 60))
|
|
bucket["last_refill"] = now
|
|
bucket["total"] += 1
|
|
|
|
# Check if allowed
|
|
if bucket["tokens"] >= 1:
|
|
bucket["tokens"] -= 1
|
|
return True, {
|
|
"remaining": int(bucket["tokens"]),
|
|
"limit": rate,
|
|
"total": bucket["total"],
|
|
}
|
|
else:
|
|
bucket["blocked"] += 1
|
|
wait_time = (1 - bucket["tokens"]) * (60 / rate)
|
|
return False, {
|
|
"remaining": 0,
|
|
"limit": rate,
|
|
"retry_after": round(wait_time, 1),
|
|
"total": bucket["total"],
|
|
"blocked": bucket["blocked"],
|
|
}
|
|
|
|
def _cleanup(self):
|
|
"""Remove stale buckets older than 1 hour."""
|
|
now = time.time()
|
|
stale = [ip for ip, b in self._buckets.items() if now - b["last_refill"] > 3600]
|
|
for ip in stale:
|
|
del self._buckets[ip]
|
|
self._last_cleanup = now
|
|
|
|
def get_stats(self) -> dict:
|
|
"""Get global rate limiter statistics."""
|
|
total_active = len(self._buckets)
|
|
total_blocked = sum(b["blocked"] for b in self._buckets.values())
|
|
total_requests = sum(b["total"] for b in self._buckets.values())
|
|
return {
|
|
"active_ips": total_active,
|
|
"total_requests": total_requests,
|
|
"total_blocked": total_blocked,
|
|
"default_rpm": self.default_rpm,
|
|
"burst": self.burst,
|
|
}
|