cryptorugmunch
17b16c8666
The AI features in llm_features.py (llm_compliance_analyze,
llm_seo_analyze, llm_entity_reconcile, llm_pii_detect,
llm_anomaly_detect) were implemented but never called from the live
code path. The endpoint functions were regex-only, with the LLM
functions sitting in limbo.
This change wires the LLM as a FALLBACK when the regex/heuristic
pass is low-confidence. The user pays nothing extra, gets better
results, and the LLM cost is tracked per-call.
Changes:
- compliance.py run_compliance_check:
When tos_result.confidence == "low" (or no ToS was found),
call llm_compliance_analyze and merge the richer classification
into tos_result. llm_enhanced: True is set.
Pass-through: the LLM fields (provider, cost, risk_summary, etc.)
are now copied into the terms_of_service sub-dict of the response.
- seo_monitor.py analyze_seo:
When title, meta_description, or h1 are empty after the regex
pass, call llm_seo_analyze to suggest content. Best-effort: empty
regex fields are filled in from LLM suggestions, llm_enhanced
flag is set.
- reconciliation.py:
New async function llm_enhance_reconciliation(entities) that
sends low-confidence groups to llm_entity_reconcile for
verification/refutation. Returns a summary dict with counts.
- New test file tests/test_llm_fallback.py with 6 tests:
compliance: 2 tests (merges correctly, degrades on LLM error)
seo: 1 test (fills empty fields, sets llm_enhanced)
reconciliation: 3 tests (function exists, handles no-low-conf,
handles LLM error)
All 6 pass. All existing compliance/seo/reconciliation tests
(28) still pass.
Defaults: the LLM uses the fleet's free Ollama on Talos
(100.100.18.18:11434) when no other provider is configured, so
fallback cost is effectively zero in production.
292 lines
10 KiB
Python
292 lines
10 KiB
Python
"""Pry — SEO Content Change Monitor.
|
|
Track competitor meta tags, titles, descriptions, headings, content changes."""
|
|
from paths import PRY_DATA_DIR
|
|
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
#
|
|
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
|
# Licensed under MIT. See LICENSE.
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from contextlib import suppress
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SEO_DIR = PRY_DATA_DIR / "seo"
|
|
SEO_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
async def analyze_seo(url: str) -> dict[str, Any]:
|
|
"""Analyze SEO elements from a URL."""
|
|
from lxml import html as lxml_html
|
|
|
|
from client import get_client
|
|
|
|
client = await get_client()
|
|
try:
|
|
resp = await client.get(
|
|
url,
|
|
timeout=30,
|
|
follow_redirects=True,
|
|
headers={"User-Agent": "Mozilla/5.0 (SEO Monitor)"},
|
|
)
|
|
if not resp.is_success:
|
|
return {"url": url, "error": f"HTTP {resp.status_code}"}
|
|
|
|
tree = lxml_html.fromstring(resp.text)
|
|
|
|
result: dict[str, Any] = {
|
|
"url": url,
|
|
"title": _get_title(tree),
|
|
"meta_description": _get_meta_content(tree, "description"),
|
|
"meta_keywords": _get_meta_content(tree, "keywords"),
|
|
"h1": _get_headings(tree, "h1"),
|
|
"h2": _get_headings(tree, "h2"),
|
|
"canonical": _get_attr(tree, 'link[rel="canonical"]', "href"),
|
|
"og_title": _get_meta_content(tree, "og:title"),
|
|
"og_description": _get_meta_content(tree, "og:description"),
|
|
"og_image": _get_meta_content(tree, "og:image"),
|
|
"twitter_title": _get_meta_content(tree, "twitter:title"),
|
|
"twitter_description": _get_meta_content(tree, "twitter:description"),
|
|
"robots_meta": _get_meta_content(tree, "robots"),
|
|
"charset": _get_charset(tree, resp),
|
|
"viewport": _get_meta_content(tree, "viewport"),
|
|
"word_count": _count_words(resp.text),
|
|
"links_internal": _count_links(tree, url, internal=True),
|
|
"links_external": _count_links(tree, url, internal=False),
|
|
"has_schema": _has_schema(resp.text),
|
|
"hreflang_tags": _get_hreflangs(tree),
|
|
"status_code": resp.status_code,
|
|
"content_type": resp.headers.get("content-type", ""),
|
|
"last_modified": resp.headers.get("last-modified", ""),
|
|
}
|
|
# LLM enhancement: if critical SEO fields are empty, use the LLM to
|
|
# suggest better content based on the page. Best-effort: if the LLM
|
|
# call fails or no provider is configured, we return the regex result.
|
|
missing_critical = [f for f in ("title", "meta_description", "h1") if not result.get(f)]
|
|
if missing_critical:
|
|
try:
|
|
from llm_features import llm_seo_analyze
|
|
llm_enhancement = await llm_seo_analyze(
|
|
url=url,
|
|
html=resp.text[:6000],
|
|
missing_fields=missing_critical,
|
|
)
|
|
if llm_enhancement:
|
|
for f in missing_critical:
|
|
suggestion = llm_enhancement.get(f)
|
|
if suggestion and not result.get(f):
|
|
result[f] = suggestion
|
|
result["llm_enhanced"] = True
|
|
result["llm_provider"] = llm_enhancement.get("llm_provider", "")
|
|
result["llm_cost_usd"] = llm_enhancement.get("llm_cost_usd", 0.0)
|
|
except Exception as e:
|
|
logger.debug("llm_seo_enhance_failed", extra={"url": url, "error": str(e)[:80]})
|
|
|
|
return result
|
|
except Exception as e:
|
|
return {"url": url, "error": str(e)[:200]}
|
|
|
|
|
|
def _get_title(tree: Any) -> str:
|
|
el = tree.find(".//title")
|
|
return el.text_content().strip()[:200] if el is not None and el.text is not None else ""
|
|
|
|
|
|
def _get_meta_content(tree: Any, name: str) -> str:
|
|
for el in tree.xpath(f'//meta[@name="{name}"] | //meta[@property="{name}"]'):
|
|
content = el.get("content", "")
|
|
return content.strip()[:500] if content else ""
|
|
return ""
|
|
|
|
|
|
def _get_headings(tree: Any, tag: str) -> list[str]:
|
|
return [
|
|
h.text_content().strip()[:200] for h in tree.xpath(f"//{tag}") if h.text_content().strip()
|
|
]
|
|
|
|
|
|
def _get_attr(tree: Any, selector: str, attr: str) -> str:
|
|
els = tree.cssselect(selector)
|
|
return str(els[0].get(attr, ""))[:500] if els else ""
|
|
|
|
|
|
def _get_charset(tree: Any, resp: Any) -> str:
|
|
meta = tree.find(".//meta[@charset]")
|
|
if meta is not None:
|
|
return str(meta.get("charset", ""))
|
|
ct = resp.headers.get("content-type", "")
|
|
m = re.search(r"charset=([\w-]+)", ct)
|
|
return m.group(1) if m else ""
|
|
|
|
|
|
def _count_words(html: str) -> int:
|
|
text = re.sub(r"<[^>]+>", " ", html)
|
|
words = re.findall(r"\w+", text)
|
|
return len(words)
|
|
|
|
|
|
def _count_links(tree: Any, base_url: str, internal: bool) -> int:
|
|
from urllib.parse import urlparse
|
|
|
|
base_domain = urlparse(base_url).netloc
|
|
count = 0
|
|
for a in tree.xpath("//a[@href]"):
|
|
href = a.get("href", "")
|
|
if href.startswith("http") or href.startswith("//"):
|
|
domain = urlparse(href).netloc
|
|
if (internal and domain == base_domain) or (not internal and domain != base_domain):
|
|
count += 1
|
|
elif internal and href.startswith("/"):
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def _has_schema(html: str) -> bool:
|
|
return bool(re.search(r'<script[^>]*type="application/ld\+json"', html))
|
|
|
|
|
|
def _get_hreflangs(tree: Any) -> list[dict[str, str]]:
|
|
tags = []
|
|
for el in tree.xpath("//link[@hreflang]"):
|
|
tags.append(
|
|
{
|
|
"hreflang": el.get("hreflang", ""),
|
|
"href": el.get("href", ""),
|
|
}
|
|
)
|
|
return tags
|
|
|
|
|
|
async def track_seo_changes(url: str) -> dict[str, Any]:
|
|
"""Track SEO changes since last analysis."""
|
|
url_hash = hashlib.sha256(url.encode()).hexdigest()[:16]
|
|
history_path = SEO_DIR / f"seo_{url_hash}.json"
|
|
|
|
# Get current SEO data
|
|
current = await analyze_seo(url)
|
|
if "error" in current:
|
|
return current
|
|
|
|
# Load previous data
|
|
previous: dict[str, Any] | None = None
|
|
if history_path.exists():
|
|
with suppress(json.JSONDecodeError, OSError):
|
|
previous = json.loads(history_path.read_text())
|
|
|
|
# Detect changes
|
|
changes = []
|
|
if previous:
|
|
tracked_fields = ["title", "meta_description", "meta_keywords", "h1", "h2", "canonical"]
|
|
for field in tracked_fields:
|
|
old_val = previous.get(field)
|
|
new_val = current.get(field)
|
|
if old_val != new_val:
|
|
changes.append(
|
|
{
|
|
"field": field,
|
|
"type": "changed",
|
|
"from": old_val,
|
|
"to": new_val,
|
|
"severity": "high" if field in ("title", "meta_description") else "medium",
|
|
}
|
|
)
|
|
|
|
# Word count change
|
|
old_words = previous.get("word_count", 0)
|
|
new_words = current.get("word_count", 0)
|
|
if old_words and new_words and abs(new_words - old_words) > 100:
|
|
changes.append(
|
|
{
|
|
"field": "word_count",
|
|
"type": "changed",
|
|
"from": old_words,
|
|
"to": new_words,
|
|
"delta": new_words - old_words,
|
|
"severity": "low",
|
|
}
|
|
)
|
|
|
|
# Save current data
|
|
with suppress(OSError):
|
|
history_path.write_text(json.dumps(current, indent=2))
|
|
|
|
return {
|
|
"url": url,
|
|
"current": current,
|
|
"changes": changes,
|
|
"change_count": len(changes),
|
|
"has_changes": len(changes) > 0,
|
|
"is_first_scan": previous is None,
|
|
"checked_at": datetime.now(UTC).isoformat(),
|
|
}
|
|
|
|
|
|
async def get_seo_keyword_insights(url: str, keywords: list[str]) -> dict[str, Any]:
|
|
"""Analyze which keywords a URL is targeting based on content analysis."""
|
|
from client import get_client
|
|
|
|
client = await get_client()
|
|
try:
|
|
resp = await client.get(url, timeout=30, follow_redirects=True)
|
|
if not resp.is_success:
|
|
return {"url": url, "error": f"HTTP {resp.status_code}"}
|
|
|
|
text = resp.text.lower()
|
|
results = []
|
|
for keyword in keywords:
|
|
kw_lower = keyword.lower().strip()
|
|
# Check in title
|
|
title = _get_title_from_html(text)
|
|
in_title = kw_lower in title.lower() if title else False
|
|
# Check in H1
|
|
h1s = re.findall(r"<h1[^>]*>(.*?)</h1>", text, re.DOTALL)
|
|
in_h1 = any(kw_lower in h.lower() for h in h1s)
|
|
# Check in meta description
|
|
meta_desc = _get_meta_from_html(text, "description")
|
|
in_meta = kw_lower in meta_desc.lower() if meta_desc else False
|
|
# Count in body
|
|
body = re.sub(r"<[^>]+>", " ", text)
|
|
frequency = body.lower().count(kw_lower)
|
|
|
|
results.append(
|
|
{
|
|
"keyword": keyword,
|
|
"frequency": frequency,
|
|
"in_title": in_title,
|
|
"in_h1": in_h1,
|
|
"in_meta_description": in_meta,
|
|
"density": round(frequency / max(len(body.split()), 1) * 100, 2)
|
|
if frequency > 0
|
|
else 0,
|
|
}
|
|
)
|
|
|
|
return {
|
|
"url": url,
|
|
"keywords_analyzed": len(keywords),
|
|
"results": results,
|
|
}
|
|
except Exception as e:
|
|
return {"url": url, "error": str(e)[:200]}
|
|
|
|
|
|
def _get_title_from_html(html: str) -> str:
|
|
m = re.search(r"<title[^>]*>(.*?)</title>", html, re.DOTALL)
|
|
return m.group(1).strip() if m else ""
|
|
|
|
|
|
def _get_meta_from_html(html: str, name: str) -> str:
|
|
m = re.search(f"<meta[^>]*name=[\"']{name}[\"'][^>]*content=[\"']([^\"']*)[\"']", html)
|
|
if m:
|
|
return m.group(1)
|
|
m = re.search(f"<meta[^>]*property=[\"']{name}[\"'][^>]*content=[\"']([^\"']*)[\"']", html)
|
|
return m.group(1) if m else ""
|