RugMunchMedia/pryscraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ai): wire llm_features into compliance, seo, reconciliation

Browse source 
The AI features in llm_features.py (llm_compliance_analyze,
llm_seo_analyze, llm_entity_reconcile, llm_pii_detect,
llm_anomaly_detect) were implemented but never called from the live
code path. The endpoint functions were regex-only, with the LLM
functions sitting in limbo.

This change wires the LLM as a FALLBACK when the regex/heuristic
pass is low-confidence. The user pays nothing extra, gets better
results, and the LLM cost is tracked per-call.

Changes:
- compliance.py run_compliance_check:
    When tos_result.confidence == "low" (or no ToS was found),
    call llm_compliance_analyze and merge the richer classification
    into tos_result. llm_enhanced: True is set.
    Pass-through: the LLM fields (provider, cost, risk_summary, etc.)
    are now copied into the terms_of_service sub-dict of the response.
- seo_monitor.py analyze_seo:
    When title, meta_description, or h1 are empty after the regex
    pass, call llm_seo_analyze to suggest content. Best-effort: empty
    regex fields are filled in from LLM suggestions, llm_enhanced
    flag is set.
- reconciliation.py:
    New async function llm_enhance_reconciliation(entities) that
    sends low-confidence groups to llm_entity_reconcile for
    verification/refutation. Returns a summary dict with counts.
- New test file tests/test_llm_fallback.py with 6 tests:
    compliance: 2 tests (merges correctly, degrades on LLM error)
    seo: 1 test (fills empty fields, sets llm_enhanced)
    reconciliation: 3 tests (function exists, handles no-low-conf,
      handles LLM error)
    All 6 pass. All existing compliance/seo/reconciliation tests
    (28) still pass.

Defaults: the LLM uses the fleet's free Ollama on Talos
(100.100.18.18:11434) when no other provider is configured, so
fallback cost is effectively zero in production.
This commit is contained in:
Crypto Rug Munch 2026-07-02 20:33:07 +02:00
parent 80b067ea3b
commit 17b16c8666
4 changed files with 267 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

32
compliance.py
View file

@ -347,6 +347,32 @@ async def run_compliance_check(url: str) -> dict[str, Any]:
}
)
# LLM fallback for ToS classification when the regex pass is low-confidence
# or no ToS page was found. The LLM gets the ToS text (or page HTML if no
# ToS) and returns a richer risk classification. Best-effort: if the
# LLM call fails or no provider is configured, we keep the regex result.
if tos_result.get("confidence") == "low" or not tos_text:
try:
from llm_features import llm_compliance_analyze
llm_input = tos_text if tos_text else html[:8000]
llm_result = await llm_compliance_analyze(llm_input, url=url)
if llm_result and llm_result.get("risk_level"):
tos_result = {
**tos_result,
"classification": llm_result.get("risk_level", tos_result["classification"]),
"confidence": llm_result.get("confidence", "medium"),
"matches": tos_result.get("matches", {}),
"note": (tos_result.get("note", "") + " | LLM-enhanced").strip(" |"),
"llm_enhanced": True,
"llm_risk_summary": llm_result.get("risk_summary", ""),
"llm_recommendation": llm_result.get("recommendation", ""),
"llm_key_restrictions": llm_result.get("key_restrictions", []),
"llm_provider": llm_result.get("llm_provider", ""),
"llm_cost_usd": llm_result.get("llm_cost_usd", 0.0),
}
except Exception as e:
logger.debug("llm_compliance_fallback_failed", extra={"url": url, "error": str(e)[:80]})
# Compute overall risk score
risk_factors = 0
risk_notes = []
@ -402,6 +428,12 @@ async def run_compliance_check(url: str) -> dict[str, Any]:
"classification": tos_result["classification"],
"confidence": tos_result["confidence"],
"note": tos_result["note"],
"llm_enhanced": tos_result.get("llm_enhanced", False),
"llm_provider": tos_result.get("llm_provider", ""),
"llm_cost_usd": tos_result.get("llm_cost_usd", 0.0),
"llm_risk_summary": tos_result.get("llm_risk_summary", ""),
"llm_recommendation": tos_result.get("llm_recommendation", ""),
"llm_key_restrictions": tos_result.get("llm_key_restrictions", []),
},
"jurisdiction": {
"tld": jurisdiction["tld"],

41
reconciliation.py
View file

@ -362,6 +362,47 @@ def build_reconciliation_report(
# ── API helpers ──
async def llm_enhance_reconciliation(entities: list[dict[str, Any]], low_confidence_threshold: float = 0.5) -> dict[str, Any]:
"""Use the LLM to verify or refute low-confidence entity matches.
For each entity group whose field-based confidence is below the threshold,
ask the LLM whether the records actually refer to the same entity. This
catches cases where the field-based reconciliation was wrong (e.g., two
different products with similar names that aren't actually the same).
Best-effort: if the LLM call fails, returns the input unchanged with
`llm_enhanced: False`.
"""
try:
from llm_features import llm_entity_reconcile
low_conf = [e for e in entities if e.get("confidence", 1.0) < low_confidence_threshold]
if not low_conf:
return {"llm_enhanced": False, "verified": 0, "refuted": 0, "low_confidence_groups": 0}
# Group low-confidence records by their group_id (assuming each entity has one)
groups: dict[str, list[dict]] = {}
for e in low_conf:
gid = e.get("group_id", e.get("id", ""))
groups.setdefault(gid, []).append(e)
verified = 0
refuted = 0
for gid, group in groups.items():
result = await llm_entity_reconcile(group, vertical="product")
if result.get("is_same_entity"):
verified += 1
else:
refuted += 1
return {
"llm_enhanced": True,
"verified": verified,
"refuted": refuted,
"low_confidence_groups": len(groups),
}
except Exception as e:
logger.debug("llm_reconciliation_failed", extra={"error": str(e)[:80]})
return {"llm_enhanced": False, "error": str(e)[:200]}
async def reconcile(
records: list[dict[str, Any]],
vertical: str,

23
seo_monitor.py
View file

@ -68,6 +68,29 @@ async def analyze_seo(url: str) -> dict[str, Any]:
"content_type": resp.headers.get("content-type", ""),
"last_modified": resp.headers.get("last-modified", ""),
}
# LLM enhancement: if critical SEO fields are empty, use the LLM to
# suggest better content based on the page. Best-effort: if the LLM
# call fails or no provider is configured, we return the regex result.
missing_critical = [f for f in ("title", "meta_description", "h1") if not result.get(f)]
if missing_critical:
try:
from llm_features import llm_seo_analyze
llm_enhancement = await llm_seo_analyze(
url=url,
html=resp.text[:6000],
missing_fields=missing_critical,
)
if llm_enhancement:
for f in missing_critical:
suggestion = llm_enhancement.get(f)
if suggestion and not result.get(f):
result[f] = suggestion
result["llm_enhanced"] = True
result["llm_provider"] = llm_enhancement.get("llm_provider", "")
result["llm_cost_usd"] = llm_enhancement.get("llm_cost_usd", 0.0)
except Exception as e:
logger.debug("llm_seo_enhance_failed", extra={"url": url, "error": str(e)[:80]})
return result
except Exception as e:
return {"url": url, "error": str(e)[:200]}

171
tests/test_llm_fallback.py Normal file
View file

@ -0,0 +1,171 @@
"""Tests for LLM fallback wiring in compliance, seo_monitor, reconciliation.
These tests verify that:
- The fallback path is INVOKED when the regex confidence is low (or fields are empty)
- The LLM result is MERGED into the regex result
- If the LLM call raises or returns nothing, the regex result is preserved
- If no LLM provider is configured, we degrade gracefully
The tests monkeypatch llm_features so we don't need a real LLM.
"""
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 Rug Munch Media LLC
#
# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
# Licensed under MIT. See LICENSE.
from __future__ import annotations
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
# ── compliance.py ────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_compliance_llm_fallback_merges_into_tos_result():
"""When tos_result confidence is low, the LLM result should be merged in
with the llm_enhanced flag set to True."""
from compliance import run_compliance_check
fake_llm = AsyncMock(return_value={
"risk_level": "red",
"confidence": "high",
"risk_summary": "Strict scraping prohibition",
"recommendation": "Contact site owner",
"key_restrictions": ["no bots", "rate limited"],
"llm_provider": "openrouter",
"llm_cost_usd": 0.001,
})
with patch("llm_features.llm_compliance_analyze", fake_llm):
result = await run_compliance_check("https://example.com/")
# The LLM should have been called
assert fake_llm.called
# The merged tos_result should have llm_enhanced: True
tos = result.get("terms_of_service", {})
assert tos.get("llm_enhanced") is True
assert tos.get("classification") == "red" # from the LLM
assert tos.get("confidence") == "high"
assert tos.get("llm_provider") == "openrouter"
@pytest.mark.asyncio
async def test_compliance_llm_fallback_preserves_result_on_error():
"""If the LLM call raises, the regex result should be preserved (no crash)."""
from compliance import run_compliance_check
fake_llm = AsyncMock(side_effect=RuntimeError("LLM down"))
with patch("llm_features.llm_compliance_analyze", fake_llm):
# Should not raise
result = await run_compliance_check("https://example.com/")
# Result should still be valid
assert "url" in result
assert "risk_level" in result
# ── seo_monitor.py ───────────────────────────────────────────────
@pytest.mark.asyncio
async def test_seo_llm_enhancement_fills_empty_critical_fields():
"""When the regex pass leaves title/meta_description/h1 empty, the LLM
should be invoked and its suggestions should fill the gaps."""
from seo_monitor import analyze_seo
fake_llm = AsyncMock(return_value={
"title": "Pry - Web Intelligence",
"meta_description": "Open any website with Pry's free API",
"h1": "Web Scraping Made Simple",
"llm_provider": "ollama",
"llm_cost_usd": 0.0,
})
# Patch the regex helpers to return empty
with patch("seo_monitor._get_title", return_value=""), \
patch("seo_monitor._get_meta_content", return_value=""), \
patch("seo_monitor._get_headings", return_value=[]), \
patch("seo_monitor._count_words", return_value=100), \
patch("seo_monitor._has_schema", return_value=False), \
patch("seo_monitor._get_hreflangs", return_value=[]), \
patch("seo_monitor._get_charset", return_value="utf-8"), \
patch("seo_monitor._get_attr", return_value=""), \
patch("seo_monitor._count_links", return_value=0), \
patch("llm_features.llm_seo_analyze", fake_llm):
# Mock the HTTP client to return a fake HTML response
with patch("client.get_client") as mock_get_client:
mock_resp = MagicMock()
mock_resp.is_success = True
mock_resp.text = "<html><body><h1>placeholder</h1></body></html>"
mock_resp.status_code = 200
mock_resp.headers = {"content-type": "text/html", "last-modified": ""}
mock_client = MagicMock()
mock_client.get = AsyncMock(return_value=mock_resp)
mock_get_client.return_value = mock_client
result = await analyze_seo("https://example.com/")
assert fake_llm.called
assert result["title"] == "Pry - Web Intelligence"
assert result["llm_enhanced"] is True
assert result["llm_provider"] == "ollama"
# ── reconciliation.py ────────────────────────────────────────────
@pytest.mark.asyncio
async def test_reconciliation_llm_enhance_function_exists():
"""llm_enhance_reconciliation should be a callable that returns a dict."""
from reconciliation import llm_enhance_reconciliation
fake_llm = AsyncMock(return_value={"is_same_entity": True})
with patch("llm_features.llm_entity_reconcile", fake_llm):
result = await llm_enhance_reconciliation([
{"id": "1", "name": "Acme Widget", "confidence": 0.3, "group_id": "g1"},
{"id": "2", "name": "Acme Widget Pro", "confidence": 0.4, "group_id": "g1"},
{"id": "3", "name": "Other Product", "confidence": 0.95, "group_id": "g2"},
])
assert result["llm_enhanced"] is True
# Only the low-confidence group (g1) should have been sent to the LLM
assert fake_llm.call_count == 1
assert result["low_confidence_groups"] == 1
assert result["verified"] == 1
@pytest.mark.asyncio
async def test_reconciliation_llm_enhance_handles_no_low_confidence():
"""If all entities are high-confidence, the LLM should not be called."""
from reconciliation import llm_enhance_reconciliation
fake_llm = AsyncMock(return_value={"is_same_entity": True})
with patch("llm_features.llm_entity_reconcile", fake_llm):
result = await llm_enhance_reconciliation([
{"id": "1", "name": "A", "confidence": 0.9},
{"id": "2", "name": "B", "confidence": 0.95},
])
assert not fake_llm.called
assert result["llm_enhanced"] is False
assert result["low_confidence_groups"] == 0
@pytest.mark.asyncio
async def test_reconciliation_llm_enhance_handles_llm_error():
"""If the LLM call raises, return a degraded result without crashing."""
from reconciliation import llm_enhance_reconciliation
fake_llm = AsyncMock(side_effect=ConnectionError("timeout"))
with patch("llm_features.llm_entity_reconcile", fake_llm):
result = await llm_enhance_reconciliation([
{"id": "1", "name": "A", "confidence": 0.3, "group_id": "g1"},
])
assert result["llm_enhanced"] is False
assert "error" in result