cryptorugmunch
8d25702eca
Squashed from chore/license-relicense. Full message preserved in the original branch commitbb77eb5. See ADR-0002 for the decision rationale. Refs: ADR-0002, commitbb77eb5
77 lines
2.5 KiB
Python
77 lines
2.5 KiB
Python
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
"""Tests for AI training data pipeline."""
|
|
|
|
from training_data import (
|
|
classify_license,
|
|
export_training_dataset,
|
|
strip_copyright_verbatim,
|
|
strip_pii,
|
|
)
|
|
|
|
|
|
def test_classify_license_mit() -> None:
|
|
text = "This project is licensed under the MIT License. It is provided as-is without warranty."
|
|
result = classify_license(text)
|
|
assert result["license"] == "mit"
|
|
assert result["tier"] == "permissive"
|
|
|
|
|
|
def test_classify_license_proprietary() -> None:
|
|
text = "Copyright 2024 Example Corp. All rights reserved. Proprietary software."
|
|
result = classify_license(text)
|
|
assert result["license"] == "proprietary"
|
|
assert result["tier"] == "restrictive"
|
|
|
|
|
|
def test_classify_license_cc0() -> None:
|
|
text = "CC0 1.0 Universal \u2014 dedicated to the public domain."
|
|
result = classify_license(text)
|
|
assert result["license"] == "cc0"
|
|
assert result["tier"] == "permissive"
|
|
|
|
|
|
def test_classify_license_unknown() -> None:
|
|
result = classify_license("Some random text with no license info.")
|
|
assert result["license"] == "unknown"
|
|
|
|
|
|
def test_strip_pii_email() -> None:
|
|
text = "Contact us at support@example.com for help."
|
|
cleaned, stats = strip_pii(text)
|
|
assert "EMAIL_REDACTED" in cleaned
|
|
assert stats.get("email") == 1
|
|
|
|
|
|
def test_strip_pii_phone() -> None:
|
|
text = "Call 555-123-4567 for support."
|
|
cleaned, stats = strip_pii(text)
|
|
assert "PHONE_REDACTED" in cleaned
|
|
assert stats.get("phone") == 1
|
|
|
|
|
|
def test_strip_pii_no_pii() -> None:
|
|
text = "This is clean text with no personal information."
|
|
cleaned, stats = strip_pii(text)
|
|
assert cleaned == text
|
|
assert all(v == 0 for v in stats.values()) or not stats
|
|
|
|
|
|
def test_strip_copyright() -> None:
|
|
text = "Some content. Copyright 2024 Example Corp. All rights reserved. More content."
|
|
cleaned, stats = strip_copyright_verbatim(text)
|
|
assert "COPYRIGHT" in cleaned or stats["blocks_removed"] >= 0
|
|
|
|
|
|
def test_export_dataset() -> None:
|
|
records = [
|
|
{"content": "Hello world", "url": "https://example.com"},
|
|
{"content": "Contact me@example.com for info", "url": "https://example.com/contact"},
|
|
]
|
|
result = export_training_dataset(records, format="jsonl", clean_room=True)
|
|
assert result["success"] is True
|
|
assert result["record_count"] == 2
|
|
assert "dataset_id" in result
|