cryptorugmunch
8d25702eca
Squashed from chore/license-relicense. Full message preserved in the original branch commitbb77eb5. See ADR-0002 for the decision rationale. Refs: ADR-0002, commitbb77eb5
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
"""Validate all scraper templates — verify JSON schema, required fields, and selector structure.
|
|
This ensures every template is structurally correct before we claim it works."""
|
|
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
TEMPLATES_DIR = Path(__file__).parent
|
|
REQUIRED_KEYS = ["name", "description", "site", "category", "icon", "schema"]
|
|
REQUIRED_SCHEMA_KEYS = ["name", "base_selector", "fields"]
|
|
REQUIRED_FIELD_KEYS = ["name", "selector", "type"]
|
|
VALID_TYPES = {"text", "attribute", "html", "nested", "count", "exists", "regex"}
|
|
VALID_TRANSFORMS = {"strip_currency", "lower", "upper", "strip", "int", "float", ""}
|
|
|
|
|
|
def validate_template(path: Path) -> list[str]:
|
|
errors = []
|
|
try:
|
|
data = json.loads(path.read_text())
|
|
except json.JSONDecodeError as e:
|
|
return [f"Invalid JSON: {e}"]
|
|
|
|
for key in REQUIRED_KEYS:
|
|
if key not in data:
|
|
errors.append(f"Missing required key: {key}")
|
|
|
|
schema = data.get("schema", {})
|
|
for key in REQUIRED_SCHEMA_KEYS:
|
|
if key not in schema:
|
|
errors.append(f"Schema missing required key: {key}")
|
|
|
|
fields = schema.get("fields", [])
|
|
if not fields:
|
|
errors.append("Schema has no fields")
|
|
|
|
for i, field in enumerate(fields):
|
|
for key in REQUIRED_FIELD_KEYS:
|
|
if key not in field:
|
|
errors.append(f"Field {i} missing required key: {key}")
|
|
ftype = field.get("type", "")
|
|
if ftype and ftype not in VALID_TYPES:
|
|
errors.append(f"Field {i} invalid type: {ftype}")
|
|
transform = field.get("transform", "")
|
|
if transform and transform not in VALID_TRANSFORMS:
|
|
errors.append(f"Field {i} invalid transform: {transform}")
|
|
|
|
return errors
|
|
|
|
|
|
def main() -> int:
|
|
all_errors = {}
|
|
total = 0
|
|
valid = 0
|
|
|
|
for path in sorted(TEMPLATES_DIR.glob("*.json")):
|
|
if path.name == "validate_templates.py":
|
|
continue
|
|
total += 1
|
|
errors = validate_template(path)
|
|
if errors:
|
|
all_errors[path.name] = errors
|
|
else:
|
|
valid += 1
|
|
|
|
if all_errors:
|
|
for name, errors in all_errors.items():
|
|
print(f"❌ {name}:")
|
|
for e in errors:
|
|
print(f" - {e}")
|
|
print(f"\n{valid}/{total} templates valid, {len(all_errors)} failed")
|
|
return 1
|
|
else:
|
|
print(f"✅ All {total} templates valid!")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|