docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
136
extractor.py
Normal file
136
extractor.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
"""Pry — JSON schema extraction engine.
|
||||
Two modes: pattern (free, no LLM) and LLM (Ollama, for complex schemas).
|
||||
LLM failures fall back gracefully to pattern mode.
|
||||
No hallucination: JSON output is always parsed and validated.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
class SchemaExtractor:
|
||||
"""Extract structured JSON data from scraped markdown content.
|
||||
Pattern mode is always tried first; LLM mode is fallback for complex schemas."""
|
||||
|
||||
def __init__(self):
|
||||
self.ollama_base = "http://100.100.18.18:11434"
|
||||
|
||||
async def extract(
|
||||
self, content: str, schema: dict[str, Any], mode: str = "auto"
|
||||
) -> dict[str, Any]:
|
||||
"""Extract fields matching the provided schema.
|
||||
Schema format: {"field_name": "description of what to extract"}
|
||||
|
||||
If LLM mode fails (Ollama down, timeout), falls back to pattern mode.
|
||||
"""
|
||||
if not content or not schema:
|
||||
return {}
|
||||
|
||||
# Pattern mode first (always works, no dependencies)
|
||||
pattern_result = self._pattern_extract(content, schema)
|
||||
|
||||
# Use LLM mode only if requested explicitly or schema is complex
|
||||
use_llm = mode == "llm" or (mode == "auto" and len(schema) > 5)
|
||||
if not use_llm:
|
||||
return pattern_result
|
||||
|
||||
# Try LLM extraction, fall back to pattern on failure
|
||||
try:
|
||||
llm_result = await self._llm_extract(content, schema)
|
||||
if llm_result and not llm_result.get("_error"):
|
||||
# Merge: LLM values override pattern, but pattern fills gaps
|
||||
merged = {**pattern_result, **llm_result}
|
||||
return {k: v for k, v in merged.items() if v is not None and v != ""}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return pattern_result
|
||||
|
||||
def _pattern_extract(self, content: str, schema: dict[str, Any]) -> dict[str, Any]:
|
||||
result = {}
|
||||
for field, desc in schema.items():
|
||||
value = self._find_value(content, field, desc)
|
||||
if value:
|
||||
result[field] = value
|
||||
return result
|
||||
|
||||
def _find_value(self, content: str, field: str, desc: str) -> str | None:
|
||||
"""Multi-strategy field extraction. Returns first match found."""
|
||||
# Strategy 1: "Label: Value" patterns
|
||||
field_variants = [field, field.replace("_", " "), field.replace("_", "")]
|
||||
for variant in field_variants:
|
||||
if not variant:
|
||||
continue
|
||||
escaped = re.escape(variant)
|
||||
m = re.search(rf"(?im){escaped}\s*[:=\-≈>]\s*(.+?)(?:\n|$)", content)
|
||||
if m:
|
||||
val = m.group(1).strip().rstrip(".,;")
|
||||
if val and len(val) < 500:
|
||||
return val
|
||||
|
||||
# Strategy 2: Context-aware patterns from description
|
||||
desc_lower = desc.lower()
|
||||
if "price" in desc_lower or "cost" in desc_lower or "usd" in desc_lower:
|
||||
m = re.search(r"[\$€£¥]?\s*[\d,]+\.?\d*\s*(?:USD|EUR|GBP)?", content)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
if "email" in desc_lower:
|
||||
m = re.search(r"[\w.+-]+@[\w-]+\.[\w.-]+", content)
|
||||
if m:
|
||||
return m.group(0)
|
||||
if "url" in desc_lower or "link" in desc_lower:
|
||||
m = re.search(r'https?://[^\s"\'<>]+', content)
|
||||
if m:
|
||||
return m.group(0)
|
||||
if "phone" in desc_lower or "telephone" in desc_lower:
|
||||
m = re.search(r"\+?\d[\d\s\-().]{7,}", content)
|
||||
if m:
|
||||
return m.group(0).strip()
|
||||
if "date" in desc_lower:
|
||||
m = re.search(r"\d{4}[-/]\d{1,2}[-/]\d{1,2}", content)
|
||||
if m:
|
||||
return m.group(0)
|
||||
if "number" in desc_lower or "count" in desc_lower or "total" in desc_lower:
|
||||
nums = re.findall(r"\b\d[\d,]*\.?\d*\b", content)
|
||||
if nums:
|
||||
return max((n for n in nums if len(n) < 20), key=len)
|
||||
|
||||
return None
|
||||
|
||||
async def _llm_extract(self, content: str, schema: dict[str, Any]) -> dict[str, Any]:
|
||||
"""LLM-guided extraction. Returns dict on success, {"_error": msg} on failure."""
|
||||
import httpx
|
||||
|
||||
schema_str = json.dumps(schema, indent=2)
|
||||
truncated = content[:8000]
|
||||
prompt = (
|
||||
"Extract the following fields from the text below.\n"
|
||||
"Return ONLY a valid JSON object with these fields — no explanation, no markdown.\n"
|
||||
f"Schema: {schema_str}\n"
|
||||
f"Text:\n{truncated}\n\nJSON:"
|
||||
)
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
f"{self.ollama_base}/api/generate",
|
||||
json={
|
||||
"model": "qwen2.5-coder:3b",
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
"options": {"num_ctx": 8192, "temperature": 0.05},
|
||||
},
|
||||
)
|
||||
data = resp.json()
|
||||
response = data.get("response", "")
|
||||
|
||||
# Extract first JSON object from response (non-greedy)
|
||||
json_match = re.search(r"\{[^{}]*\}", response, re.S)
|
||||
if json_match:
|
||||
obj = json.loads(json_match.group(0))
|
||||
if isinstance(obj, dict):
|
||||
return obj
|
||||
return {"_raw": response[:500]}
|
||||
except Exception as e:
|
||||
return {"_error": str(e)}
|
||||
Loading…
Add table
Add a link
Reference in a new issue