docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
133
schema_extraction.py
Normal file
133
schema_extraction.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Pry — Schema.org / JSON-LD Auto-Extraction.
|
||||
Most modern sites (e-commerce, news, recipes, events) embed structured data
|
||||
as JSON-LD in <script type=\"application/ld+json\"> tags. Extract this directly
|
||||
instead of parsing HTML — 100x faster and more accurate."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, ClassVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SchemaExtractor:
|
||||
"""Extract structured data from Schema.org / JSON-LD / Microdata / RDFa."""
|
||||
|
||||
SCHEMA_TYPES: ClassVar[dict[str, list[str]]] = {
|
||||
"Product": ["name", "description", "brand", "sku", "mpn", "image", "offers"],
|
||||
"Article": ["headline", "author", "datePublished", "image", "publisher"],
|
||||
"NewsArticle": ["headline", "author", "datePublished", "articleBody"],
|
||||
"Recipe": ["name", "recipeIngredient", "recipeInstructions", "cookTime"],
|
||||
"Event": ["name", "startDate", "endDate", "location", "offers"],
|
||||
"Organization": ["name", "url", "logo", "address", "contactPoint"],
|
||||
"Person": ["name", "jobTitle", "worksFor", "sameAs"],
|
||||
"LocalBusiness": ["name", "address", "telephone", "openingHours", "priceRange"],
|
||||
"Review": ["author", "datePublished", "reviewBody", "reviewRating"],
|
||||
"VideoObject": ["name", "description", "thumbnailUrl", "uploadDate", "duration"],
|
||||
}
|
||||
|
||||
_JSONLD_RE = re.compile(
|
||||
r'<script\s+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
|
||||
def extract_jsonld(self, html: str) -> list[dict[str, Any]]:
|
||||
"""Extract all JSON-LD blocks from HTML."""
|
||||
results: list[dict[str, Any]] = []
|
||||
for match in self._JSONLD_RE.finditer(html):
|
||||
raw = match.group(1)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
cleaned = re.sub(r"/\*.*?\*/", "", raw, flags=re.DOTALL)
|
||||
try:
|
||||
data = json.loads(cleaned)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
logger.debug("jsonld_parse_failed", extra={"snippet": raw[:80]})
|
||||
continue
|
||||
if isinstance(data, list):
|
||||
results.extend(d for d in data if isinstance(d, dict))
|
||||
elif isinstance(data, dict):
|
||||
if "@graph" in data and isinstance(data["@graph"], list):
|
||||
results.extend(d for d in data["@graph"] if isinstance(d, dict))
|
||||
else:
|
||||
results.append(data)
|
||||
return results
|
||||
|
||||
def extract_microdata(self, html: str) -> list[dict[str, Any]]:
|
||||
"""Extract Microdata from HTML (itemtype, itemprop)."""
|
||||
from lxml import html as lxml_html
|
||||
|
||||
tree = lxml_html.fromstring(html)
|
||||
results: list[dict[str, Any]] = []
|
||||
for elem in tree.xpath('//*[@itemtype]'):
|
||||
itemtype = elem.get("itemtype") or ""
|
||||
item: dict[str, Any] = {"@type": itemtype.split("/")[-1]}
|
||||
for prop in elem.xpath('.//*[@itemprop]'):
|
||||
key = prop.get("itemprop")
|
||||
if not key:
|
||||
continue
|
||||
value = (
|
||||
prop.get("content")
|
||||
or prop.get("href")
|
||||
or (prop.text_content().strip() if prop.text_content() else "")
|
||||
)
|
||||
if value:
|
||||
item[key] = value
|
||||
results.append(item)
|
||||
return results
|
||||
|
||||
def extract_rdfa(self, html: str) -> list[dict[str, Any]]:
|
||||
"""Extract RDFa attributes from HTML."""
|
||||
from lxml import html as lxml_html
|
||||
|
||||
tree = lxml_html.fromstring(html)
|
||||
results: list[dict[str, Any]] = []
|
||||
for elem in tree.xpath('//*[@typeof]'):
|
||||
item: dict[str, Any] = {"@type": elem.get("typeof")}
|
||||
for prop in elem.xpath('.//*[@property]'):
|
||||
raw_key = prop.get("property") or ""
|
||||
key = raw_key.split(":")[-1]
|
||||
value = prop.get("content") or prop.text_content().strip()
|
||||
if key and value:
|
||||
item[key] = value
|
||||
results.append(item)
|
||||
return results
|
||||
|
||||
def extract_all(self, html: str) -> dict[str, Any]:
|
||||
"""Extract all structured data from HTML."""
|
||||
jsonld = self.extract_jsonld(html)
|
||||
microdata = self.extract_microdata(html)
|
||||
rdfa = self.extract_rdfa(html)
|
||||
normalized = [self._normalize(item) for item in jsonld + microdata + rdfa]
|
||||
return {
|
||||
"jsonld": jsonld,
|
||||
"microdata": microdata,
|
||||
"rdfa": rdfa,
|
||||
"normalized": normalized,
|
||||
"count": len(normalized),
|
||||
}
|
||||
|
||||
def _normalize(self, item: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize schema item to common format."""
|
||||
schema_type = item.get("@type", item.get("type", "Unknown"))
|
||||
if "@context" in item:
|
||||
source = "jsonld"
|
||||
elif "itemtype" in str(item) or any(
|
||||
isinstance(v, str) and v.startswith("https://schema.org/") for v in item.values()
|
||||
):
|
||||
source = "microdata"
|
||||
elif "typeof" in str(item):
|
||||
source = "rdfa"
|
||||
else:
|
||||
source = "unknown"
|
||||
return {
|
||||
"type": schema_type,
|
||||
"source": source,
|
||||
"data": {
|
||||
k: v
|
||||
for k, v in item.items()
|
||||
if k not in ("@context", "itemtype", "typeof", "itemprop", "property")
|
||||
},
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue