docs: apply fleet-template (16-artifact scaffold)

Adds missing standard artifacts:
- README.md (if missing)
- AGENTS.md (AI agent contract)
- PLAN.md (current sprint)
- STATUS.md (where we are)
- DEVELOPMENT.md (dev workflow)
- DEPLOYMENT.md (deploy procedure)
- TESTING.md (test strategy)
- DECISIONS.md (ADR index + templates)
- .github/CODEOWNERS
- .github/workflows/ci.yml

Preserves all existing artifacts.

Refs: RugMunchMedia/fleet-template
This commit is contained in:
Crypto Rug Munch 2026-07-02 02:07:13 +07:00
commit 47ba268131
310 changed files with 38429 additions and 0 deletions

138
template_engine.py Normal file
View file

@ -0,0 +1,138 @@
"""Pry — Template Engine for one-click site-specific scraping.
Loads and executes pre-built extraction schemas for popular websites."""
import json
import logging
from pathlib import Path
from typing import Any, cast
from extraction import JsonCssExtractionStrategy
logger = logging.getLogger(__name__)
TEMPLATES_DIR = Path(__file__).parent / "templates"
TEMPLATES_DIR.mkdir(parents=True, exist_ok=True)
def list_templates() -> list[dict[str, Any]]:
"""List all available scraper templates with metadata."""
templates = []
for path in sorted(TEMPLATES_DIR.glob("*.json")):
try:
data = json.loads(path.read_text())
templates.append(
{
"id": path.stem,
"name": data.get("name", path.stem),
"description": data.get("description", ""),
"site": data.get("site", ""),
"category": data.get("category", "general"),
"fields": list(data.get("schema", {}).get("fields", [])),
"icon": data.get("icon", "🌐"),
}
)
except (json.JSONDecodeError, OSError):
continue
return templates
def get_template(template_id: str) -> dict[str, Any] | None:
"""Get a specific template by ID."""
path = TEMPLATES_DIR / f"{template_id}.json"
if not path.exists():
return None
try:
return cast(dict[str, Any], json.loads(path.read_text()))
except (json.JSONDecodeError, OSError):
return None
async def execute_template(template_id: str, url: str, **options: Any) -> dict[str, Any]:
"""Execute a scraper template against a URL.
This scrapes the URL and extracts structured data using the template's schema.
"""
template = get_template(template_id)
if not template:
return {"success": False, "error": f"Template not found: {template_id}"}
schema = template.get("schema", {})
if not schema:
return {"success": False, "error": "Template has no extraction schema"}
# Scrape the URL with progressive fallback
from scraper import PryScraper
scraper = PryScraper()
html = ""
scrape_error = ""
# Helper: get raw HTML from scraper result
def _get_html(r):
for key in ("raw_html", "html", "content"):
val = r.get(key)
if val:
return val
return ""
# Tier 1: Try with Cloudflare bypass
result = await scraper.scrape(url, {"bypass_cloudflare": True})
if result.get("status") == "ok":
html = _get_html(result)
else:
scrape_error = result.get("error", "")
# Quick extraction test — if no items found and JS render available, retry
if html:
strategy = JsonCssExtractionStrategy(schema)
items = strategy.extract(html)
if not items and len(html) > 500:
result_js = await scraper.scrape(url, {"bypass_cloudflare": True, "js_render": True})
if result_js.get("status") == "ok":
html_js = _get_html(result_js)
if html_js and len(html_js) > len(html):
html = html_js
else:
# Tier 2: Try with JS rendering
result_js = await scraper.scrape(url, {"bypass_cloudflare": True, "js_render": True})
if result_js.get("status") == "ok":
html = _get_html(result_js)
elif not scrape_error:
scrape_error = result_js.get("error", "Scrape failed")
if not html:
return {"success": False, "error": scrape_error or "No HTML content scraped"}
# Extract using the schema
strategy = JsonCssExtractionStrategy(schema)
items = strategy.extract(html)
if not items:
return {
"success": True,
"data": {
"template_id": template_id,
"template_name": template.get("name", ""),
"url": url,
"items": [],
"count": 0,
"warning": "No items matched the CSS selectors. The page structure may have changed.",
"extracted_at": __import__("datetime")
.datetime.now(__import__("datetime").timezone.utc)
.isoformat(),
},
}
return {
"success": True,
"data": {
"template_id": template_id,
"template_name": template.get("name", ""),
"url": url,
"items": items,
"count": len(items),
"extracted_at": __import__("datetime")
.datetime.now(__import__("datetime").timezone.utc)
.isoformat(),
},
}