docs: apply fleet-template (16-artifact scaffold)
Adds missing standard artifacts: - README.md (if missing) - AGENTS.md (AI agent contract) - PLAN.md (current sprint) - STATUS.md (where we are) - DEVELOPMENT.md (dev workflow) - DEPLOYMENT.md (deploy procedure) - TESTING.md (test strategy) - DECISIONS.md (ADR index + templates) - .github/CODEOWNERS - .github/workflows/ci.yml Preserves all existing artifacts. Refs: RugMunchMedia/fleet-template
This commit is contained in:
commit
47ba268131
310 changed files with 38429 additions and 0 deletions
397
quality.py
Normal file
397
quality.py
Normal file
|
|
@ -0,0 +1,397 @@
|
|||
"""Pry — Data Quality SLA Dashboard.
|
||||
Per-extraction quality metrics, anomaly detection, freshness tracking."""
|
||||
|
||||
import difflib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
QUALITY_DIR = Path(os.path.expanduser("~/.pry/quality"))
|
||||
QUALITY_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ── Quality Metrics ──
|
||||
|
||||
|
||||
def compute_completeness(
|
||||
data: dict[str, Any] | list[Any], schema: dict[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Compute completeness metrics for extracted data.
|
||||
|
||||
Measures:
|
||||
- Field presence: what % of expected fields have non-null values
|
||||
- Record count: expected vs actual for list data
|
||||
- Schema adherence: what % of fields match expected types
|
||||
"""
|
||||
if isinstance(data, list):
|
||||
return _compute_list_completeness(data, schema)
|
||||
return _compute_dict_completeness(data, schema)
|
||||
|
||||
|
||||
def _compute_dict_completeness(
|
||||
data: dict[str, Any], schema: dict[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Compute completeness for a single dict."""
|
||||
if not data:
|
||||
return {
|
||||
"score": 0,
|
||||
"total_fields": 0,
|
||||
"filled_fields": 0,
|
||||
"null_fields": 0,
|
||||
"empty_fields": 0,
|
||||
}
|
||||
|
||||
total = len(data)
|
||||
filled = sum(1 for v in data.values() if v not in (None, "", [], {}))
|
||||
nulls = sum(1 for v in data.values() if v is None)
|
||||
empties = sum(1 for v in data.values() if v in ("", [], {}) and v is not None)
|
||||
|
||||
score = round(filled / total * 100, 1) if total > 0 else 0.0
|
||||
|
||||
return {
|
||||
"score": score,
|
||||
"total_fields": total,
|
||||
"filled_fields": filled,
|
||||
"null_fields": nulls,
|
||||
"empty_fields": empties,
|
||||
}
|
||||
|
||||
|
||||
def _compute_list_completeness(
|
||||
data: list[Any], schema: dict[str, Any] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""Compute completeness for a list of records."""
|
||||
if not data:
|
||||
return {"score": 0, "record_count": 0, "avg_record_score": 0}
|
||||
|
||||
record_scores = []
|
||||
total_possible = 0
|
||||
total_filled = 0
|
||||
|
||||
for record in data:
|
||||
if isinstance(record, dict):
|
||||
result = _compute_dict_completeness(record, schema)
|
||||
record_scores.append(result["score"])
|
||||
total_possible += result["total_fields"]
|
||||
total_filled += result["filled_fields"]
|
||||
|
||||
avg_record = round(sum(record_scores) / len(record_scores), 1) if record_scores else 0.0
|
||||
overall = round(total_filled / total_possible * 100, 1) if total_possible > 0 else 0.0
|
||||
|
||||
return {
|
||||
"score": overall,
|
||||
"record_count": len(data),
|
||||
"avg_record_score": avg_record,
|
||||
"min_record_score": min(record_scores) if record_scores else 0,
|
||||
"max_record_score": max(record_scores) if record_scores else 0,
|
||||
}
|
||||
|
||||
|
||||
def compute_schema_adherence(
|
||||
data: dict[str, Any] | list[dict[str, Any]],
|
||||
expected_types: dict[str, type],
|
||||
) -> dict[str, Any]:
|
||||
"""Check if fields match expected types."""
|
||||
records = data if isinstance(data, list) else [data]
|
||||
field_issues: dict[str, int] = {}
|
||||
total_checks = 0
|
||||
type_mismatches = 0
|
||||
|
||||
for record in records:
|
||||
if isinstance(record, dict):
|
||||
for field, expected_type in expected_types.items():
|
||||
total_checks += 1
|
||||
value = record.get(field)
|
||||
if value is not None and not isinstance(value, expected_type):
|
||||
type_mismatches += 1
|
||||
field_issues[field] = field_issues.get(field, 0) + 1
|
||||
|
||||
return {
|
||||
"score": round((total_checks - type_mismatches) / total_checks * 100, 1)
|
||||
if total_checks > 0
|
||||
else 100.0,
|
||||
"total_checks": total_checks,
|
||||
"type_mismatches": type_mismatches,
|
||||
"field_issues": field_issues,
|
||||
}
|
||||
|
||||
|
||||
def compute_null_rate(data: dict[str, Any] | list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Compute null/empty rate per field."""
|
||||
records = data if isinstance(data, list) else [data]
|
||||
field_stats: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for record in records:
|
||||
if isinstance(record, dict):
|
||||
for key, value in record.items():
|
||||
if key not in field_stats:
|
||||
field_stats[key] = {"total": 0, "null": 0, "empty": 0}
|
||||
field_stats[key]["total"] += 1
|
||||
if value is None:
|
||||
field_stats[key]["null"] += 1
|
||||
elif value in ("", [], {}):
|
||||
field_stats[key]["empty"] += 1
|
||||
|
||||
return {
|
||||
field: {
|
||||
"null_rate": round(stats["null"] / stats["total"] * 100, 1),
|
||||
"empty_rate": round(stats["empty"] / stats["total"] * 100, 1),
|
||||
"total": stats["total"],
|
||||
}
|
||||
for field, stats in field_stats.items()
|
||||
}
|
||||
|
||||
|
||||
def compute_freshness(data: dict[str, Any], max_age_seconds: int = 3600) -> dict[str, Any]:
|
||||
"""Check data freshness against expected age."""
|
||||
now = time.time()
|
||||
timestamps = []
|
||||
|
||||
# Look for timestamp fields
|
||||
for key in ["timestamp", "checked_at", "cached_at", "scraped_at", "created_at", "updated_at"]:
|
||||
val = data.get(key)
|
||||
if val:
|
||||
timestamps.append(val)
|
||||
|
||||
if not timestamps:
|
||||
return {
|
||||
"fresh": True,
|
||||
"age_seconds": None,
|
||||
"note": "No timestamp field found in data — cannot verify freshness",
|
||||
}
|
||||
|
||||
# Try to parse the first timestamp
|
||||
ts = timestamps[0]
|
||||
if isinstance(ts, (int, float)):
|
||||
age = now - ts
|
||||
elif isinstance(ts, str):
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts)
|
||||
age = now - dt.timestamp()
|
||||
except (ValueError, TypeError):
|
||||
return {
|
||||
"fresh": True,
|
||||
"age_seconds": None,
|
||||
"note": f"Could not parse timestamp: {ts[:30]}",
|
||||
}
|
||||
else:
|
||||
return {"fresh": True, "age_seconds": None, "note": "Timestamp in unknown format"}
|
||||
|
||||
return {
|
||||
"fresh": age <= max_age_seconds,
|
||||
"age_seconds": round(age),
|
||||
"max_age_seconds": max_age_seconds,
|
||||
"note": f"Data is {_format_age(age)} old",
|
||||
}
|
||||
|
||||
|
||||
def _format_age(seconds: float) -> str:
|
||||
if seconds < 60:
|
||||
return f"{int(seconds)}s"
|
||||
elif seconds < 3600:
|
||||
return f"{int(seconds / 60)}m"
|
||||
elif seconds < 86400:
|
||||
return f"{int(seconds / 3600)}h"
|
||||
return f"{int(seconds / 86400)}d"
|
||||
|
||||
|
||||
def detect_anomalies(
|
||||
current_data: dict[str, Any],
|
||||
previous_data: dict[str, Any] | None = None,
|
||||
z_score_threshold: float = 2.0,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Detect anomalies in extracted data compared to previous runs.
|
||||
|
||||
Flags:
|
||||
- Missing fields (field present before, absent now)
|
||||
- New fields (field absent before, present now)
|
||||
- Value type changes
|
||||
- Large value swings (for numeric fields)
|
||||
- Empty results when previous had data
|
||||
"""
|
||||
anomalies: list[dict[str, Any]] = []
|
||||
|
||||
if previous_data is None:
|
||||
return []
|
||||
|
||||
# Check for empty results
|
||||
if not current_data and previous_data:
|
||||
anomalies.append(
|
||||
{
|
||||
"type": "empty_result",
|
||||
"severity": "critical",
|
||||
"field": "*",
|
||||
"message": "Extraction returned empty — previous run had data",
|
||||
}
|
||||
)
|
||||
return anomalies
|
||||
|
||||
if not isinstance(current_data, dict) or not isinstance(previous_data, dict):
|
||||
return anomalies
|
||||
|
||||
prev_keys = set(previous_data.keys())
|
||||
curr_keys = set(current_data.keys())
|
||||
|
||||
# Missing fields
|
||||
missing = prev_keys - curr_keys
|
||||
for field in missing:
|
||||
anomalies.append(
|
||||
{
|
||||
"type": "missing_field",
|
||||
"severity": "high",
|
||||
"field": field,
|
||||
"message": f"Field '{field}' was present in previous extraction but is now missing",
|
||||
}
|
||||
)
|
||||
|
||||
# New fields
|
||||
new_fields = curr_keys - prev_keys
|
||||
for field in new_fields:
|
||||
anomalies.append(
|
||||
{
|
||||
"type": "new_field",
|
||||
"severity": "info",
|
||||
"field": field,
|
||||
"message": f"New field '{field}' appeared",
|
||||
}
|
||||
)
|
||||
|
||||
# Value changes for common fields
|
||||
common = prev_keys & curr_keys
|
||||
for field in common:
|
||||
prev_val = previous_data[field]
|
||||
curr_val = current_data[field]
|
||||
|
||||
# Type change
|
||||
if prev_val is not None and curr_val is not None and type(prev_val) is not type(curr_val):
|
||||
anomalies.append(
|
||||
{
|
||||
"type": "type_change",
|
||||
"severity": "high",
|
||||
"field": field,
|
||||
"message": f"Field '{field}' changed type: {type(prev_val).__name__} → {type(curr_val).__name__}",
|
||||
"previous_type": type(prev_val).__name__,
|
||||
"current_type": type(curr_val).__name__,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Numeric swing
|
||||
if (
|
||||
isinstance(prev_val, (int, float))
|
||||
and isinstance(curr_val, (int, float))
|
||||
and prev_val != 0
|
||||
):
|
||||
z_score = abs((curr_val - prev_val) / max(abs(prev_val), 0.01))
|
||||
if z_score > z_score_threshold:
|
||||
pct = ((curr_val - prev_val) / abs(prev_val)) * 100
|
||||
anomalies.append(
|
||||
{
|
||||
"type": "value_swing",
|
||||
"severity": "high" if abs(pct) > 50 else "medium",
|
||||
"field": field,
|
||||
"message": f"Field '{field}' changed by {pct:+.1f}% ({prev_val} → {curr_val})",
|
||||
"previous_value": prev_val,
|
||||
"current_value": curr_val,
|
||||
"change_pct": round(pct, 1),
|
||||
}
|
||||
)
|
||||
|
||||
# Text content change
|
||||
if (
|
||||
isinstance(prev_val, str)
|
||||
and isinstance(curr_val, str)
|
||||
and prev_val != curr_val
|
||||
and len(prev_val) > 20
|
||||
):
|
||||
ratio = difflib.SequenceMatcher(None, prev_val, curr_val).ratio()
|
||||
if ratio < 0.5:
|
||||
anomalies.append(
|
||||
{
|
||||
"type": "content_drift",
|
||||
"severity": "medium",
|
||||
"field": field,
|
||||
"message": f"Field '{field}' content changed significantly ({round(ratio * 100)}% similarity)",
|
||||
"similarity": round(ratio, 3),
|
||||
}
|
||||
)
|
||||
|
||||
return anomalies
|
||||
|
||||
|
||||
async def run_quality_check(
|
||||
url: str,
|
||||
data: dict[str, Any],
|
||||
schema: dict[str, Any] | None = None,
|
||||
expected_types: dict[str, type] | None = None,
|
||||
max_age_seconds: int = 3600,
|
||||
) -> dict[str, Any]:
|
||||
"""Run full quality check on extracted data.
|
||||
|
||||
Returns completeness, schema adherence, freshness, anomalies.
|
||||
"""
|
||||
# Load previous data for comparison
|
||||
url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
|
||||
history_path = QUALITY_DIR / f"{url_hash}.json"
|
||||
previous_data: dict[str, Any] | None = None
|
||||
if history_path.exists():
|
||||
try:
|
||||
prev = json.loads(history_path.read_text())
|
||||
previous_data = prev.get("data")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
# Compute metrics
|
||||
completeness = compute_completeness(data, schema)
|
||||
freshness = compute_freshness(data, max_age_seconds)
|
||||
anomalies = detect_anomalies(data, previous_data)
|
||||
|
||||
null_rate: dict[str, Any] = {}
|
||||
schema_adherence: dict[str, Any] = {
|
||||
"score": 100.0,
|
||||
"total_checks": 0,
|
||||
"type_mismatches": 0,
|
||||
"field_issues": {},
|
||||
}
|
||||
|
||||
if isinstance(data, dict):
|
||||
null_rate = compute_null_rate(data)
|
||||
if expected_types:
|
||||
schema_adherence = compute_schema_adherence(data, expected_types)
|
||||
|
||||
# Save current data for future comparison
|
||||
with suppress(OSError):
|
||||
history_path.write_text(
|
||||
json.dumps({"url": url, "data": data, "checked_at": datetime.now(UTC).isoformat()})
|
||||
)
|
||||
|
||||
# Compute overall quality score (weighted average)
|
||||
quality_score = round(
|
||||
completeness.get("score", 0) * 0.4
|
||||
+ (100 - len(anomalies) * 10) * 0.3
|
||||
+ schema_adherence.get("score", 100) * 0.2
|
||||
+ (100 if freshness.get("fresh") else 50) * 0.1,
|
||||
1,
|
||||
)
|
||||
quality_score = max(0, min(100, quality_score))
|
||||
|
||||
return {
|
||||
"url": url,
|
||||
"quality_score": quality_score,
|
||||
"completeness": completeness,
|
||||
"schema_adherence": schema_adherence,
|
||||
"freshness": freshness,
|
||||
"null_rates": null_rate,
|
||||
"anomalies": anomalies,
|
||||
"anomaly_count": len(anomalies),
|
||||
"critical_anomalies": sum(1 for a in anomalies if a["severity"] == "critical"),
|
||||
"has_previous_data": previous_data is not None,
|
||||
"checked_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue