docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
80
cache.py
Normal file
80
cache.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Pry — in-memory LRU cache with Redis persistence.
|
||||
Prevents re-scraping the same URL within TTL window."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
class ResponseCache:
|
||||
"""Two-tier cache: local LRU + optional Redis persistence.
|
||||
|
||||
Default TTL: 300 seconds (5 min) for pages, 3600 (1 hr) for API calls.
|
||||
"""
|
||||
|
||||
def __init__(self, capacity: int = 500, redis_url: str = None):
|
||||
self.capacity = capacity
|
||||
self._cache: OrderedDict = OrderedDict()
|
||||
self._redis = None
|
||||
self._redis_url = redis_url
|
||||
self._hits = 0
|
||||
self._misses = 0
|
||||
|
||||
def _key(self, url: str, options: dict | None = None) -> str:
|
||||
"""Generate cache key from URL + options."""
|
||||
opt_str = json.dumps(options or {}, sort_keys=True)
|
||||
raw = f"{url}:{opt_str}"
|
||||
return hashlib.md5(raw.encode()).hexdigest()
|
||||
|
||||
def get(self, url: str, options: dict | None = None) -> dict | None:
|
||||
"""Get cached response if fresh."""
|
||||
key = self._key(url, options)
|
||||
now = time.time()
|
||||
|
||||
# Check local cache
|
||||
if key in self._cache:
|
||||
entry = self._cache[key]
|
||||
if entry["expires"] > now:
|
||||
self._hits += 1
|
||||
self._cache.move_to_end(key)
|
||||
return entry["data"]
|
||||
else:
|
||||
del self._cache[key]
|
||||
|
||||
self._misses += 1
|
||||
return None
|
||||
|
||||
def set(self, url: str, data: dict, options: dict | None = None, ttl: int = 300):
|
||||
"""Cache a response with TTL in seconds."""
|
||||
key = self._key(url, options)
|
||||
entry = {"data": data, "expires": time.time() + ttl, "cached_at": time.time()}
|
||||
|
||||
# Evict oldest if at capacity
|
||||
if len(self._cache) >= self.capacity:
|
||||
self._cache.popitem(last=False)
|
||||
|
||||
self._cache[key] = entry
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""Get cache hit/miss statistics."""
|
||||
total = self._hits + self._misses
|
||||
return {
|
||||
"size": len(self._cache),
|
||||
"capacity": self.capacity,
|
||||
"hits": self._hits,
|
||||
"misses": self._misses,
|
||||
"hit_rate": round(self._hits / total * 100, 1) if total > 0 else 0,
|
||||
}
|
||||
|
||||
def invalidate(self, url: str):
|
||||
"""Remove cached entry for a URL."""
|
||||
keys_to_remove = [k for k, v in self._cache.items() if v["data"].get("url") == url]
|
||||
for k in keys_to_remove:
|
||||
del self._cache[k]
|
||||
|
||||
def clear(self):
|
||||
"""Clear entire cache."""
|
||||
self._cache.clear()
|
||||
self._hits = 0
|
||||
self._misses = 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue