diff --git a/compliance.py b/compliance.py index e06e564..71dd995 100644 --- a/compliance.py +++ b/compliance.py @@ -347,6 +347,32 @@ async def run_compliance_check(url: str) -> dict[str, Any]: } ) + # LLM fallback for ToS classification when the regex pass is low-confidence + # or no ToS page was found. The LLM gets the ToS text (or page HTML if no + # ToS) and returns a richer risk classification. Best-effort: if the + # LLM call fails or no provider is configured, we keep the regex result. + if tos_result.get("confidence") == "low" or not tos_text: + try: + from llm_features import llm_compliance_analyze + llm_input = tos_text if tos_text else html[:8000] + llm_result = await llm_compliance_analyze(llm_input, url=url) + if llm_result and llm_result.get("risk_level"): + tos_result = { + **tos_result, + "classification": llm_result.get("risk_level", tos_result["classification"]), + "confidence": llm_result.get("confidence", "medium"), + "matches": tos_result.get("matches", {}), + "note": (tos_result.get("note", "") + " | LLM-enhanced").strip(" |"), + "llm_enhanced": True, + "llm_risk_summary": llm_result.get("risk_summary", ""), + "llm_recommendation": llm_result.get("recommendation", ""), + "llm_key_restrictions": llm_result.get("key_restrictions", []), + "llm_provider": llm_result.get("llm_provider", ""), + "llm_cost_usd": llm_result.get("llm_cost_usd", 0.0), + } + except Exception as e: + logger.debug("llm_compliance_fallback_failed", extra={"url": url, "error": str(e)[:80]}) + # Compute overall risk score risk_factors = 0 risk_notes = [] @@ -402,6 +428,12 @@ async def run_compliance_check(url: str) -> dict[str, Any]: "classification": tos_result["classification"], "confidence": tos_result["confidence"], "note": tos_result["note"], + "llm_enhanced": tos_result.get("llm_enhanced", False), + "llm_provider": tos_result.get("llm_provider", ""), + "llm_cost_usd": tos_result.get("llm_cost_usd", 0.0), + "llm_risk_summary": tos_result.get("llm_risk_summary", ""), + "llm_recommendation": tos_result.get("llm_recommendation", ""), + "llm_key_restrictions": tos_result.get("llm_key_restrictions", []), }, "jurisdiction": { "tld": jurisdiction["tld"], diff --git a/reconciliation.py b/reconciliation.py index c69eb5f..7cb5802 100644 --- a/reconciliation.py +++ b/reconciliation.py @@ -362,6 +362,47 @@ def build_reconciliation_report( # ── API helpers ── + + +async def llm_enhance_reconciliation(entities: list[dict[str, Any]], low_confidence_threshold: float = 0.5) -> dict[str, Any]: + """Use the LLM to verify or refute low-confidence entity matches. + + For each entity group whose field-based confidence is below the threshold, + ask the LLM whether the records actually refer to the same entity. This + catches cases where the field-based reconciliation was wrong (e.g., two + different products with similar names that aren't actually the same). + + Best-effort: if the LLM call fails, returns the input unchanged with + `llm_enhanced: False`. + """ + try: + from llm_features import llm_entity_reconcile + low_conf = [e for e in entities if e.get("confidence", 1.0) < low_confidence_threshold] + if not low_conf: + return {"llm_enhanced": False, "verified": 0, "refuted": 0, "low_confidence_groups": 0} + # Group low-confidence records by their group_id (assuming each entity has one) + groups: dict[str, list[dict]] = {} + for e in low_conf: + gid = e.get("group_id", e.get("id", "")) + groups.setdefault(gid, []).append(e) + verified = 0 + refuted = 0 + for gid, group in groups.items(): + result = await llm_entity_reconcile(group, vertical="product") + if result.get("is_same_entity"): + verified += 1 + else: + refuted += 1 + return { + "llm_enhanced": True, + "verified": verified, + "refuted": refuted, + "low_confidence_groups": len(groups), + } + except Exception as e: + logger.debug("llm_reconciliation_failed", extra={"error": str(e)[:80]}) + return {"llm_enhanced": False, "error": str(e)[:200]} + async def reconcile( records: list[dict[str, Any]], vertical: str, diff --git a/seo_monitor.py b/seo_monitor.py index e65053b..464c566 100644 --- a/seo_monitor.py +++ b/seo_monitor.py @@ -68,6 +68,29 @@ async def analyze_seo(url: str) -> dict[str, Any]: "content_type": resp.headers.get("content-type", ""), "last_modified": resp.headers.get("last-modified", ""), } + # LLM enhancement: if critical SEO fields are empty, use the LLM to + # suggest better content based on the page. Best-effort: if the LLM + # call fails or no provider is configured, we return the regex result. + missing_critical = [f for f in ("title", "meta_description", "h1") if not result.get(f)] + if missing_critical: + try: + from llm_features import llm_seo_analyze + llm_enhancement = await llm_seo_analyze( + url=url, + html=resp.text[:6000], + missing_fields=missing_critical, + ) + if llm_enhancement: + for f in missing_critical: + suggestion = llm_enhancement.get(f) + if suggestion and not result.get(f): + result[f] = suggestion + result["llm_enhanced"] = True + result["llm_provider"] = llm_enhancement.get("llm_provider", "") + result["llm_cost_usd"] = llm_enhancement.get("llm_cost_usd", 0.0) + except Exception as e: + logger.debug("llm_seo_enhance_failed", extra={"url": url, "error": str(e)[:80]}) + return result except Exception as e: return {"url": url, "error": str(e)[:200]} diff --git a/tests/test_llm_fallback.py b/tests/test_llm_fallback.py new file mode 100644 index 0000000..3b846cd --- /dev/null +++ b/tests/test_llm_fallback.py @@ -0,0 +1,171 @@ +"""Tests for LLM fallback wiring in compliance, seo_monitor, reconciliation. + +These tests verify that: +- The fallback path is INVOKED when the regex confidence is low (or fields are empty) +- The LLM result is MERGED into the regex result +- If the LLM call raises or returns nothing, the regex result is preserved +- If no LLM provider is configured, we degrade gracefully + +The tests monkeypatch llm_features so we don't need a real LLM. +""" +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Rug Munch Media LLC +# +# Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper +# Licensed under MIT. See LICENSE. +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + + +# ── compliance.py ──────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_compliance_llm_fallback_merges_into_tos_result(): + """When tos_result confidence is low, the LLM result should be merged in + with the llm_enhanced flag set to True.""" + from compliance import run_compliance_check + + fake_llm = AsyncMock(return_value={ + "risk_level": "red", + "confidence": "high", + "risk_summary": "Strict scraping prohibition", + "recommendation": "Contact site owner", + "key_restrictions": ["no bots", "rate limited"], + "llm_provider": "openrouter", + "llm_cost_usd": 0.001, + }) + + with patch("llm_features.llm_compliance_analyze", fake_llm): + result = await run_compliance_check("https://example.com/") + + # The LLM should have been called + assert fake_llm.called + # The merged tos_result should have llm_enhanced: True + tos = result.get("terms_of_service", {}) + assert tos.get("llm_enhanced") is True + assert tos.get("classification") == "red" # from the LLM + assert tos.get("confidence") == "high" + assert tos.get("llm_provider") == "openrouter" + + +@pytest.mark.asyncio +async def test_compliance_llm_fallback_preserves_result_on_error(): + """If the LLM call raises, the regex result should be preserved (no crash).""" + from compliance import run_compliance_check + + fake_llm = AsyncMock(side_effect=RuntimeError("LLM down")) + + with patch("llm_features.llm_compliance_analyze", fake_llm): + # Should not raise + result = await run_compliance_check("https://example.com/") + + # Result should still be valid + assert "url" in result + assert "risk_level" in result + + +# ── seo_monitor.py ─────────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_seo_llm_enhancement_fills_empty_critical_fields(): + """When the regex pass leaves title/meta_description/h1 empty, the LLM + should be invoked and its suggestions should fill the gaps.""" + from seo_monitor import analyze_seo + + fake_llm = AsyncMock(return_value={ + "title": "Pry - Web Intelligence", + "meta_description": "Open any website with Pry's free API", + "h1": "Web Scraping Made Simple", + "llm_provider": "ollama", + "llm_cost_usd": 0.0, + }) + + # Patch the regex helpers to return empty + with patch("seo_monitor._get_title", return_value=""), \ + patch("seo_monitor._get_meta_content", return_value=""), \ + patch("seo_monitor._get_headings", return_value=[]), \ + patch("seo_monitor._count_words", return_value=100), \ + patch("seo_monitor._has_schema", return_value=False), \ + patch("seo_monitor._get_hreflangs", return_value=[]), \ + patch("seo_monitor._get_charset", return_value="utf-8"), \ + patch("seo_monitor._get_attr", return_value=""), \ + patch("seo_monitor._count_links", return_value=0), \ + patch("llm_features.llm_seo_analyze", fake_llm): + # Mock the HTTP client to return a fake HTML response + with patch("client.get_client") as mock_get_client: + mock_resp = MagicMock() + mock_resp.is_success = True + mock_resp.text = "

placeholder

" + mock_resp.status_code = 200 + mock_resp.headers = {"content-type": "text/html", "last-modified": ""} + mock_client = MagicMock() + mock_client.get = AsyncMock(return_value=mock_resp) + mock_get_client.return_value = mock_client + + result = await analyze_seo("https://example.com/") + + assert fake_llm.called + assert result["title"] == "Pry - Web Intelligence" + assert result["llm_enhanced"] is True + assert result["llm_provider"] == "ollama" + + +# ── reconciliation.py ──────────────────────────────────────────── + +@pytest.mark.asyncio +async def test_reconciliation_llm_enhance_function_exists(): + """llm_enhance_reconciliation should be a callable that returns a dict.""" + from reconciliation import llm_enhance_reconciliation + + fake_llm = AsyncMock(return_value={"is_same_entity": True}) + + with patch("llm_features.llm_entity_reconcile", fake_llm): + result = await llm_enhance_reconciliation([ + {"id": "1", "name": "Acme Widget", "confidence": 0.3, "group_id": "g1"}, + {"id": "2", "name": "Acme Widget Pro", "confidence": 0.4, "group_id": "g1"}, + {"id": "3", "name": "Other Product", "confidence": 0.95, "group_id": "g2"}, + ]) + + assert result["llm_enhanced"] is True + # Only the low-confidence group (g1) should have been sent to the LLM + assert fake_llm.call_count == 1 + assert result["low_confidence_groups"] == 1 + assert result["verified"] == 1 + + +@pytest.mark.asyncio +async def test_reconciliation_llm_enhance_handles_no_low_confidence(): + """If all entities are high-confidence, the LLM should not be called.""" + from reconciliation import llm_enhance_reconciliation + + fake_llm = AsyncMock(return_value={"is_same_entity": True}) + + with patch("llm_features.llm_entity_reconcile", fake_llm): + result = await llm_enhance_reconciliation([ + {"id": "1", "name": "A", "confidence": 0.9}, + {"id": "2", "name": "B", "confidence": 0.95}, + ]) + + assert not fake_llm.called + assert result["llm_enhanced"] is False + assert result["low_confidence_groups"] == 0 + + +@pytest.mark.asyncio +async def test_reconciliation_llm_enhance_handles_llm_error(): + """If the LLM call raises, return a degraded result without crashing.""" + from reconciliation import llm_enhance_reconciliation + + fake_llm = AsyncMock(side_effect=ConnectionError("timeout")) + + with patch("llm_features.llm_entity_reconcile", fake_llm): + result = await llm_enhance_reconciliation([ + {"id": "1", "name": "A", "confidence": 0.3, "group_id": "g1"}, + ]) + + assert result["llm_enhanced"] is False + assert "error" in result