docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
172
cookie_warmer.py
Normal file
172
cookie_warmer.py
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
"""Pry — Cookie Warming and Session Aging.
|
||||
Pre-age cookies by browsing legitimate pages first. Aged cookies with
|
||||
realistic browsing history bypass anti-bot detection that checks for
|
||||
'fresh' cookies. This is a technique used by professional scraping services."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WARMER_DIR = Path(__file__).parent / "warmed_cookies"
|
||||
WARMER_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
# Realistic pages to "warm" cookies against (these are common referrers/browsing paths)
|
||||
WARMING_PAGES = {
|
||||
"amazon": ["https://www.amazon.com/", "https://www.amazon.com/gp/help/customer/contact-us",
|
||||
"https://www.amazon.com/privacy", "https://www.amazon.com/conditions-of-use"],
|
||||
"ebay": ["https://www.ebay.com/", "https://www.ebay.com/help/home",
|
||||
"https://www.ebay.com/myp/PurchaseHistory"],
|
||||
"shopify": ["https://www.shopify.com/", "https://www.shopify.com/pricing"],
|
||||
"twitter": ["https://twitter.com/", "https://twitter.com/explore",
|
||||
"https://twitter.com/settings/account"],
|
||||
"linkedin": ["https://www.linkedin.com/", "https://www.linkedin.com/help/intro"],
|
||||
"generic": ["https://www.google.com/", "https://en.wikipedia.org/wiki/Main_Page",
|
||||
"https://github.com/"],
|
||||
}
|
||||
|
||||
|
||||
class CookieWarmer:
|
||||
"""Warm cookies by simulating realistic user browsing before scraping."""
|
||||
|
||||
def __init__(self, browser_pool: Any = None):
|
||||
self._browser_pool = browser_pool
|
||||
|
||||
async def warm_for_site(
|
||||
self,
|
||||
target_domain: str,
|
||||
session_id: str = "",
|
||||
pages_to_visit: int = 3,
|
||||
min_delay: float = 2.0,
|
||||
max_delay: float = 8.0,
|
||||
) -> dict[str, Any]:
|
||||
"""Warm cookies for a target domain by visiting legitimate pages first.
|
||||
|
||||
Args:
|
||||
target_domain: Domain to warm cookies for (e.g., "amazon.com")
|
||||
session_id: Session identifier (for storing warmed cookies)
|
||||
pages_to_visit: Number of pages to visit to warm cookies
|
||||
min_delay: Minimum delay between page visits (seconds)
|
||||
max_delay: Maximum delay between page visits (seconds)
|
||||
"""
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
if not session_id:
|
||||
session_id = f"{target_domain}_{int(time.time())}"
|
||||
|
||||
# Find warming pages for this domain
|
||||
domain_key = next((k for k in WARMING_PAGES if k in target_domain.lower()), "generic")
|
||||
pages = WARMING_PAGES[domain_key][:pages_to_visit]
|
||||
|
||||
cookies: list[dict[str, Any]] = []
|
||||
user_agent = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
try:
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(headless=True, args=["--no-sandbox"])
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
user_agent=user_agent,
|
||||
)
|
||||
# Add stealth
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
||||
Object.defineProperty(navigator, 'plugins', {get: () => [1,2,3,4,5]});
|
||||
""")
|
||||
page = await context.new_page()
|
||||
|
||||
for warming_url in pages:
|
||||
try:
|
||||
await page.goto(warming_url, wait_until="domcontentloaded", timeout=30000)
|
||||
# Human-like behavior: scroll, hover
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight * 0.3)")
|
||||
await page.wait_for_timeout(random.randint(500, 2000))
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
await page.wait_for_timeout(random.randint(500, 1500))
|
||||
delay = random.uniform(min_delay, max_delay)
|
||||
await asyncio.sleep(delay)
|
||||
except Exception as e:
|
||||
logger.warning("warm_page_failed url=%s err=%s", warming_url, str(e)[:50])
|
||||
|
||||
# Collect cookies
|
||||
storage_cookies = await context.cookies()
|
||||
cookies = [
|
||||
{
|
||||
"name": c["name"],
|
||||
"value": c["value"],
|
||||
"domain": c["domain"],
|
||||
"path": c["path"],
|
||||
"expires": c.get("expires", -1),
|
||||
"httpOnly": c.get("httpOnly", False),
|
||||
"secure": c.get("secure", False),
|
||||
}
|
||||
for c in storage_cookies
|
||||
]
|
||||
|
||||
await browser.close()
|
||||
|
||||
# Save warmed cookies
|
||||
cookie_file = WARMER_DIR / f"{session_id}.json"
|
||||
cookie_data = {
|
||||
"domain": target_domain,
|
||||
"session_id": session_id,
|
||||
"warmed_at": datetime.now(UTC).isoformat(),
|
||||
"expires_at": (datetime.now(UTC) + timedelta(days=30)).isoformat(),
|
||||
"pages_visited": pages,
|
||||
"cookies": cookies,
|
||||
"user_agent": user_agent,
|
||||
}
|
||||
cookie_file.write_text(json.dumps(cookie_data, indent=2))
|
||||
return {
|
||||
"success": True,
|
||||
"session_id": session_id,
|
||||
"domain": target_domain,
|
||||
"cookie_count": len(cookies),
|
||||
"pages_visited": pages,
|
||||
"expires_at": cookie_data["expires_at"],
|
||||
}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)[:300]}
|
||||
|
||||
def get_warmed_cookies(self, session_id: str) -> dict[str, Any] | None:
|
||||
"""Get warmed cookies for a session."""
|
||||
cookie_file = WARMER_DIR / f"{session_id}.json"
|
||||
if not cookie_file.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(cookie_file.read_text())
|
||||
# Check expiry
|
||||
expires = datetime.fromisoformat(data["expires_at"])
|
||||
if expires < datetime.now(UTC):
|
||||
return None
|
||||
return data
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def list_sessions(self) -> list[dict[str, Any]]:
|
||||
sessions: list[dict[str, Any]] = []
|
||||
for f in WARMER_DIR.glob("*.json"):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
sessions.append(
|
||||
{
|
||||
"session_id": data["session_id"],
|
||||
"domain": data["domain"],
|
||||
"warmed_at": data["warmed_at"],
|
||||
"expires_at": data["expires_at"],
|
||||
"cookie_count": len(data.get("cookies", [])),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return sessions
|
||||
Loading…
Add table
Add a link
Reference in a new issue