docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
138
template_engine.py
Normal file
138
template_engine.py
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
"""Pry — Template Engine for one-click site-specific scraping.
|
||||
Loads and executes pre-built extraction schemas for popular websites."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from extraction import JsonCssExtractionStrategy
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TEMPLATES_DIR = Path(__file__).parent / "templates"
|
||||
TEMPLATES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def list_templates() -> list[dict[str, Any]]:
|
||||
"""List all available scraper templates with metadata."""
|
||||
templates = []
|
||||
for path in sorted(TEMPLATES_DIR.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(path.read_text())
|
||||
templates.append(
|
||||
{
|
||||
"id": path.stem,
|
||||
"name": data.get("name", path.stem),
|
||||
"description": data.get("description", ""),
|
||||
"site": data.get("site", ""),
|
||||
"category": data.get("category", "general"),
|
||||
"fields": list(data.get("schema", {}).get("fields", [])),
|
||||
"icon": data.get("icon", "🌐"),
|
||||
}
|
||||
)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
continue
|
||||
return templates
|
||||
|
||||
|
||||
def get_template(template_id: str) -> dict[str, Any] | None:
|
||||
"""Get a specific template by ID."""
|
||||
path = TEMPLATES_DIR / f"{template_id}.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return cast(dict[str, Any], json.loads(path.read_text()))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
async def execute_template(template_id: str, url: str, **options: Any) -> dict[str, Any]:
|
||||
"""Execute a scraper template against a URL.
|
||||
|
||||
This scrapes the URL and extracts structured data using the template's schema.
|
||||
"""
|
||||
template = get_template(template_id)
|
||||
if not template:
|
||||
return {"success": False, "error": f"Template not found: {template_id}"}
|
||||
|
||||
schema = template.get("schema", {})
|
||||
if not schema:
|
||||
return {"success": False, "error": "Template has no extraction schema"}
|
||||
|
||||
# Scrape the URL with progressive fallback
|
||||
from scraper import PryScraper
|
||||
|
||||
scraper = PryScraper()
|
||||
html = ""
|
||||
scrape_error = ""
|
||||
|
||||
# Helper: get raw HTML from scraper result
|
||||
def _get_html(r):
|
||||
for key in ("raw_html", "html", "content"):
|
||||
val = r.get(key)
|
||||
if val:
|
||||
return val
|
||||
return ""
|
||||
|
||||
# Tier 1: Try with Cloudflare bypass
|
||||
result = await scraper.scrape(url, {"bypass_cloudflare": True})
|
||||
if result.get("status") == "ok":
|
||||
html = _get_html(result)
|
||||
else:
|
||||
scrape_error = result.get("error", "")
|
||||
|
||||
# Quick extraction test — if no items found and JS render available, retry
|
||||
if html:
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
items = strategy.extract(html)
|
||||
if not items and len(html) > 500:
|
||||
result_js = await scraper.scrape(url, {"bypass_cloudflare": True, "js_render": True})
|
||||
if result_js.get("status") == "ok":
|
||||
html_js = _get_html(result_js)
|
||||
if html_js and len(html_js) > len(html):
|
||||
html = html_js
|
||||
else:
|
||||
# Tier 2: Try with JS rendering
|
||||
result_js = await scraper.scrape(url, {"bypass_cloudflare": True, "js_render": True})
|
||||
if result_js.get("status") == "ok":
|
||||
html = _get_html(result_js)
|
||||
elif not scrape_error:
|
||||
scrape_error = result_js.get("error", "Scrape failed")
|
||||
|
||||
if not html:
|
||||
return {"success": False, "error": scrape_error or "No HTML content scraped"}
|
||||
|
||||
# Extract using the schema
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
items = strategy.extract(html)
|
||||
|
||||
if not items:
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"template_id": template_id,
|
||||
"template_name": template.get("name", ""),
|
||||
"url": url,
|
||||
"items": [],
|
||||
"count": 0,
|
||||
"warning": "No items matched the CSS selectors. The page structure may have changed.",
|
||||
"extracted_at": __import__("datetime")
|
||||
.datetime.now(__import__("datetime").timezone.utc)
|
||||
.isoformat(),
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"data": {
|
||||
"template_id": template_id,
|
||||
"template_name": template.get("name", ""),
|
||||
"url": url,
|
||||
"items": items,
|
||||
"count": len(items),
|
||||
"extracted_at": __import__("datetime")
|
||||
.datetime.now(__import__("datetime").timezone.utc)
|
||||
.isoformat(),
|
||||
},
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue