cryptorugmunch
17b16c8666
The AI features in llm_features.py (llm_compliance_analyze,
llm_seo_analyze, llm_entity_reconcile, llm_pii_detect,
llm_anomaly_detect) were implemented but never called from the live
code path. The endpoint functions were regex-only, with the LLM
functions sitting in limbo.
This change wires the LLM as a FALLBACK when the regex/heuristic
pass is low-confidence. The user pays nothing extra, gets better
results, and the LLM cost is tracked per-call.
Changes:
- compliance.py run_compliance_check:
When tos_result.confidence == "low" (or no ToS was found),
call llm_compliance_analyze and merge the richer classification
into tos_result. llm_enhanced: True is set.
Pass-through: the LLM fields (provider, cost, risk_summary, etc.)
are now copied into the terms_of_service sub-dict of the response.
- seo_monitor.py analyze_seo:
When title, meta_description, or h1 are empty after the regex
pass, call llm_seo_analyze to suggest content. Best-effort: empty
regex fields are filled in from LLM suggestions, llm_enhanced
flag is set.
- reconciliation.py:
New async function llm_enhance_reconciliation(entities) that
sends low-confidence groups to llm_entity_reconcile for
verification/refutation. Returns a summary dict with counts.
- New test file tests/test_llm_fallback.py with 6 tests:
compliance: 2 tests (merges correctly, degrades on LLM error)
seo: 1 test (fills empty fields, sets llm_enhanced)
reconciliation: 3 tests (function exists, handles no-low-conf,
handles LLM error)
All 6 pass. All existing compliance/seo/reconciliation tests
(28) still pass.
Defaults: the LLM uses the fleet's free Ollama on Talos
(100.100.18.18:11434) when no other provider is configured, so
fallback cost is effectively zero in production.
171 lines
6.8 KiB
Python
171 lines
6.8 KiB
Python
"""Tests for LLM fallback wiring in compliance, seo_monitor, reconciliation.
|
|
|
|
These tests verify that:
|
|
- The fallback path is INVOKED when the regex confidence is low (or fields are empty)
|
|
- The LLM result is MERGED into the regex result
|
|
- If the LLM call raises or returns nothing, the regex result is preserved
|
|
- If no LLM provider is configured, we degrade gracefully
|
|
|
|
The tests monkeypatch llm_features so we don't need a real LLM.
|
|
"""
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
# ── compliance.py ────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_compliance_llm_fallback_merges_into_tos_result():
|
|
"""When tos_result confidence is low, the LLM result should be merged in
|
|
with the llm_enhanced flag set to True."""
|
|
from compliance import run_compliance_check
|
|
|
|
fake_llm = AsyncMock(return_value={
|
|
"risk_level": "red",
|
|
"confidence": "high",
|
|
"risk_summary": "Strict scraping prohibition",
|
|
"recommendation": "Contact site owner",
|
|
"key_restrictions": ["no bots", "rate limited"],
|
|
"llm_provider": "openrouter",
|
|
"llm_cost_usd": 0.001,
|
|
})
|
|
|
|
with patch("llm_features.llm_compliance_analyze", fake_llm):
|
|
result = await run_compliance_check("https://example.com/")
|
|
|
|
# The LLM should have been called
|
|
assert fake_llm.called
|
|
# The merged tos_result should have llm_enhanced: True
|
|
tos = result.get("terms_of_service", {})
|
|
assert tos.get("llm_enhanced") is True
|
|
assert tos.get("classification") == "red" # from the LLM
|
|
assert tos.get("confidence") == "high"
|
|
assert tos.get("llm_provider") == "openrouter"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_compliance_llm_fallback_preserves_result_on_error():
|
|
"""If the LLM call raises, the regex result should be preserved (no crash)."""
|
|
from compliance import run_compliance_check
|
|
|
|
fake_llm = AsyncMock(side_effect=RuntimeError("LLM down"))
|
|
|
|
with patch("llm_features.llm_compliance_analyze", fake_llm):
|
|
# Should not raise
|
|
result = await run_compliance_check("https://example.com/")
|
|
|
|
# Result should still be valid
|
|
assert "url" in result
|
|
assert "risk_level" in result
|
|
|
|
|
|
# ── seo_monitor.py ───────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_seo_llm_enhancement_fills_empty_critical_fields():
|
|
"""When the regex pass leaves title/meta_description/h1 empty, the LLM
|
|
should be invoked and its suggestions should fill the gaps."""
|
|
from seo_monitor import analyze_seo
|
|
|
|
fake_llm = AsyncMock(return_value={
|
|
"title": "Pry - Web Intelligence",
|
|
"meta_description": "Open any website with Pry's free API",
|
|
"h1": "Web Scraping Made Simple",
|
|
"llm_provider": "ollama",
|
|
"llm_cost_usd": 0.0,
|
|
})
|
|
|
|
# Patch the regex helpers to return empty
|
|
with patch("seo_monitor._get_title", return_value=""), \
|
|
patch("seo_monitor._get_meta_content", return_value=""), \
|
|
patch("seo_monitor._get_headings", return_value=[]), \
|
|
patch("seo_monitor._count_words", return_value=100), \
|
|
patch("seo_monitor._has_schema", return_value=False), \
|
|
patch("seo_monitor._get_hreflangs", return_value=[]), \
|
|
patch("seo_monitor._get_charset", return_value="utf-8"), \
|
|
patch("seo_monitor._get_attr", return_value=""), \
|
|
patch("seo_monitor._count_links", return_value=0), \
|
|
patch("llm_features.llm_seo_analyze", fake_llm):
|
|
# Mock the HTTP client to return a fake HTML response
|
|
with patch("client.get_client") as mock_get_client:
|
|
mock_resp = MagicMock()
|
|
mock_resp.is_success = True
|
|
mock_resp.text = "<html><body><h1>placeholder</h1></body></html>"
|
|
mock_resp.status_code = 200
|
|
mock_resp.headers = {"content-type": "text/html", "last-modified": ""}
|
|
mock_client = MagicMock()
|
|
mock_client.get = AsyncMock(return_value=mock_resp)
|
|
mock_get_client.return_value = mock_client
|
|
|
|
result = await analyze_seo("https://example.com/")
|
|
|
|
assert fake_llm.called
|
|
assert result["title"] == "Pry - Web Intelligence"
|
|
assert result["llm_enhanced"] is True
|
|
assert result["llm_provider"] == "ollama"
|
|
|
|
|
|
# ── reconciliation.py ────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconciliation_llm_enhance_function_exists():
|
|
"""llm_enhance_reconciliation should be a callable that returns a dict."""
|
|
from reconciliation import llm_enhance_reconciliation
|
|
|
|
fake_llm = AsyncMock(return_value={"is_same_entity": True})
|
|
|
|
with patch("llm_features.llm_entity_reconcile", fake_llm):
|
|
result = await llm_enhance_reconciliation([
|
|
{"id": "1", "name": "Acme Widget", "confidence": 0.3, "group_id": "g1"},
|
|
{"id": "2", "name": "Acme Widget Pro", "confidence": 0.4, "group_id": "g1"},
|
|
{"id": "3", "name": "Other Product", "confidence": 0.95, "group_id": "g2"},
|
|
])
|
|
|
|
assert result["llm_enhanced"] is True
|
|
# Only the low-confidence group (g1) should have been sent to the LLM
|
|
assert fake_llm.call_count == 1
|
|
assert result["low_confidence_groups"] == 1
|
|
assert result["verified"] == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconciliation_llm_enhance_handles_no_low_confidence():
|
|
"""If all entities are high-confidence, the LLM should not be called."""
|
|
from reconciliation import llm_enhance_reconciliation
|
|
|
|
fake_llm = AsyncMock(return_value={"is_same_entity": True})
|
|
|
|
with patch("llm_features.llm_entity_reconcile", fake_llm):
|
|
result = await llm_enhance_reconciliation([
|
|
{"id": "1", "name": "A", "confidence": 0.9},
|
|
{"id": "2", "name": "B", "confidence": 0.95},
|
|
])
|
|
|
|
assert not fake_llm.called
|
|
assert result["llm_enhanced"] is False
|
|
assert result["low_confidence_groups"] == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconciliation_llm_enhance_handles_llm_error():
|
|
"""If the LLM call raises, return a degraded result without crashing."""
|
|
from reconciliation import llm_enhance_reconciliation
|
|
|
|
fake_llm = AsyncMock(side_effect=ConnectionError("timeout"))
|
|
|
|
with patch("llm_features.llm_entity_reconcile", fake_llm):
|
|
result = await llm_enhance_reconciliation([
|
|
{"id": "1", "name": "A", "confidence": 0.3, "group_id": "g1"},
|
|
])
|
|
|
|
assert result["llm_enhanced"] is False
|
|
assert "error" in result
|