docs: apply fleet-template (16-artifact scaffold)

Adds missing standard artifacts:
- README.md (if missing)
- AGENTS.md (AI agent contract)
- PLAN.md (current sprint)
- STATUS.md (where we are)
- DEVELOPMENT.md (dev workflow)
- DEPLOYMENT.md (deploy procedure)
- TESTING.md (test strategy)
- DECISIONS.md (ADR index + templates)
- .github/CODEOWNERS
- .github/workflows/ci.yml

Preserves all existing artifacts.

Refs: RugMunchMedia/fleet-template
This commit is contained in:
Crypto Rug Munch 2026-07-02 02:07:13 +07:00
commit 47ba268131
310 changed files with 38429 additions and 0 deletions

133
schema_extraction.py Normal file
View file

@ -0,0 +1,133 @@
"""Pry — Schema.org / JSON-LD Auto-Extraction.
Most modern sites (e-commerce, news, recipes, events) embed structured data
as JSON-LD in <script type=\"application/ld+json\"> tags. Extract this directly
instead of parsing HTML 100x faster and more accurate."""
import json
import logging
import re
from typing import Any, ClassVar
logger = logging.getLogger(__name__)
class SchemaExtractor:
"""Extract structured data from Schema.org / JSON-LD / Microdata / RDFa."""
SCHEMA_TYPES: ClassVar[dict[str, list[str]]] = {
"Product": ["name", "description", "brand", "sku", "mpn", "image", "offers"],
"Article": ["headline", "author", "datePublished", "image", "publisher"],
"NewsArticle": ["headline", "author", "datePublished", "articleBody"],
"Recipe": ["name", "recipeIngredient", "recipeInstructions", "cookTime"],
"Event": ["name", "startDate", "endDate", "location", "offers"],
"Organization": ["name", "url", "logo", "address", "contactPoint"],
"Person": ["name", "jobTitle", "worksFor", "sameAs"],
"LocalBusiness": ["name", "address", "telephone", "openingHours", "priceRange"],
"Review": ["author", "datePublished", "reviewBody", "reviewRating"],
"VideoObject": ["name", "description", "thumbnailUrl", "uploadDate", "duration"],
}
_JSONLD_RE = re.compile(
r'<script\s+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
re.DOTALL | re.IGNORECASE,
)
def extract_jsonld(self, html: str) -> list[dict[str, Any]]:
"""Extract all JSON-LD blocks from HTML."""
results: list[dict[str, Any]] = []
for match in self._JSONLD_RE.finditer(html):
raw = match.group(1)
try:
data = json.loads(raw)
except json.JSONDecodeError:
cleaned = re.sub(r"/\*.*?\*/", "", raw, flags=re.DOTALL)
try:
data = json.loads(cleaned)
except (json.JSONDecodeError, ValueError):
logger.debug("jsonld_parse_failed", extra={"snippet": raw[:80]})
continue
if isinstance(data, list):
results.extend(d for d in data if isinstance(d, dict))
elif isinstance(data, dict):
if "@graph" in data and isinstance(data["@graph"], list):
results.extend(d for d in data["@graph"] if isinstance(d, dict))
else:
results.append(data)
return results
def extract_microdata(self, html: str) -> list[dict[str, Any]]:
"""Extract Microdata from HTML (itemtype, itemprop)."""
from lxml import html as lxml_html
tree = lxml_html.fromstring(html)
results: list[dict[str, Any]] = []
for elem in tree.xpath('//*[@itemtype]'):
itemtype = elem.get("itemtype") or ""
item: dict[str, Any] = {"@type": itemtype.split("/")[-1]}
for prop in elem.xpath('.//*[@itemprop]'):
key = prop.get("itemprop")
if not key:
continue
value = (
prop.get("content")
or prop.get("href")
or (prop.text_content().strip() if prop.text_content() else "")
)
if value:
item[key] = value
results.append(item)
return results
def extract_rdfa(self, html: str) -> list[dict[str, Any]]:
"""Extract RDFa attributes from HTML."""
from lxml import html as lxml_html
tree = lxml_html.fromstring(html)
results: list[dict[str, Any]] = []
for elem in tree.xpath('//*[@typeof]'):
item: dict[str, Any] = {"@type": elem.get("typeof")}
for prop in elem.xpath('.//*[@property]'):
raw_key = prop.get("property") or ""
key = raw_key.split(":")[-1]
value = prop.get("content") or prop.text_content().strip()
if key and value:
item[key] = value
results.append(item)
return results
def extract_all(self, html: str) -> dict[str, Any]:
"""Extract all structured data from HTML."""
jsonld = self.extract_jsonld(html)
microdata = self.extract_microdata(html)
rdfa = self.extract_rdfa(html)
normalized = [self._normalize(item) for item in jsonld + microdata + rdfa]
return {
"jsonld": jsonld,
"microdata": microdata,
"rdfa": rdfa,
"normalized": normalized,
"count": len(normalized),
}
def _normalize(self, item: dict[str, Any]) -> dict[str, Any]:
"""Normalize schema item to common format."""
schema_type = item.get("@type", item.get("type", "Unknown"))
if "@context" in item:
source = "jsonld"
elif "itemtype" in str(item) or any(
isinstance(v, str) and v.startswith("https://schema.org/") for v in item.values()
):
source = "microdata"
elif "typeof" in str(item):
source = "rdfa"
else:
source = "unknown"
return {
"type": schema_type,
"source": source,
"data": {
k: v
for k, v in item.items()
if k not in ("@context", "itemtype", "typeof", "itemprop", "property")
},
}