bb77eb5f35
Re-license Pry from full Proprietary to a dual-license model: - Core engine, extraction, templates (80+), MCP server, x402 payment rail, CLI, SDK, browser extension, WordPress plugin, Shopify app, and llm_providers: MIT (see LICENSE) - Anti-detection / stealth subset (15 files): BSL 1.1 with Change Date 2029-01-01 (see LICENSE-BSL-STEALTH) BSL files (anti-detection moat): ultimate_scraper.py, stealth_engine.py, stealth_scripts/*.js (6), camoufox_integration.py, tls_fingerprint.py, cookie_warmer.py, behavioral_biometrics.py, adaptive.py, browser_pool.py, network.py, captcha_solver.py, shadow_dom.py, lazy_load.py, signup_automator.py, auth_connector.py This enables community contributions to the core engine (templates, integrations, MCP tools) while protecting the anti-detection techniques that constitute the actual competitive moat. BSL Additional Use Grant permits free non-production use; production deployment requires a commercial license from enterprise@rugmunch.io. Changes: - Replace proprietary LICENSE with MIT LICENSE + new LICENSE-BSL-STEALTH - Add SPDX-License-Identifier headers to 300+ source files - Add docs/adr/0002-dual-licensing.md (ADR documenting the decision) - Update README.md: new License section with BSL Additional Use Grant - Update LICENSING_PRICING_STRATEGY.md: Section 3 (PryScraper) for dual license - Update AGENTS.md: license line in header + new rule 8 (PRs touching BSL rejected) - Update pyproject.toml: license = "MIT AND BSL-1.1" + classifiers + license-files - Update DECISIONS.md index with ADR-0002 - Update STATUS.md (2026-07-03) and PLAN.md sprint goals Refs: ADR-0002
86 lines
2.7 KiB
Python
86 lines
2.7 KiB
Python
"""Pry — in-memory LRU cache with Redis persistence.
|
|
Prevents re-scraping the same URL within TTL window."""
|
|
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from collections import OrderedDict
|
|
|
|
|
|
class ResponseCache:
|
|
"""Two-tier cache: local LRU + optional Redis persistence.
|
|
|
|
Default TTL: 300 seconds (5 min) for pages, 3600 (1 hr) for API calls.
|
|
"""
|
|
|
|
def __init__(self, capacity: int = 500, redis_url: str = None):
|
|
self.capacity = capacity
|
|
self._cache: OrderedDict = OrderedDict()
|
|
self._redis = None
|
|
self._redis_url = redis_url
|
|
self._hits = 0
|
|
self._misses = 0
|
|
|
|
def _key(self, url: str, options: dict | None = None) -> str:
|
|
"""Generate cache key from URL + options."""
|
|
opt_str = json.dumps(options or {}, sort_keys=True)
|
|
raw = f"{url}:{opt_str}"
|
|
return hashlib.md5(raw.encode()).hexdigest()
|
|
|
|
def get(self, url: str, options: dict | None = None) -> dict | None:
|
|
"""Get cached response if fresh."""
|
|
key = self._key(url, options)
|
|
now = time.time()
|
|
|
|
# Check local cache
|
|
if key in self._cache:
|
|
entry = self._cache[key]
|
|
if entry["expires"] > now:
|
|
self._hits += 1
|
|
self._cache.move_to_end(key)
|
|
return entry["data"]
|
|
else:
|
|
del self._cache[key]
|
|
|
|
self._misses += 1
|
|
return None
|
|
|
|
def set(self, url: str, data: dict, options: dict | None = None, ttl: int = 300):
|
|
"""Cache a response with TTL in seconds."""
|
|
key = self._key(url, options)
|
|
entry = {"data": data, "expires": time.time() + ttl, "cached_at": time.time()}
|
|
|
|
# Evict oldest if at capacity
|
|
if len(self._cache) >= self.capacity:
|
|
self._cache.popitem(last=False)
|
|
|
|
self._cache[key] = entry
|
|
|
|
def stats(self) -> dict:
|
|
"""Get cache hit/miss statistics."""
|
|
total = self._hits + self._misses
|
|
return {
|
|
"size": len(self._cache),
|
|
"capacity": self.capacity,
|
|
"hits": self._hits,
|
|
"misses": self._misses,
|
|
"hit_rate": round(self._hits / total * 100, 1) if total > 0 else 0,
|
|
}
|
|
|
|
def invalidate(self, url: str):
|
|
"""Remove cached entry for a URL."""
|
|
keys_to_remove = [k for k, v in self._cache.items() if v["data"].get("url") == url]
|
|
for k in keys_to_remove:
|
|
del self._cache[k]
|
|
|
|
def clear(self):
|
|
"""Clear entire cache."""
|
|
self._cache.clear()
|
|
self._hits = 0
|
|
self._misses = 0
|