cryptorugmunch
9db3f26f95
Some checks failed
- AST-extract all remaining routes from api.py into routers/*.py - Group routes by OpenAPI tag (AI, Advanced, Agency, ..., x402) - Keep existing auth/health/scraping/templates routers; add *_api.py for remaining routes of those tags - Move shared helpers/models/variables with the routes that need them - Update tests/test_api.py to import vision helpers from routers.vision - Regenerate openapi.json (186 paths) - All 497 tests pass; ruff clean
199 lines
7.7 KiB
Python
199 lines
7.7 KiB
Python
"""Pry — Analysis router (remaining api.py routes).
|
|
|
|
Auto-extracted from api.py during the router-split refactor.
|
|
"""
|
|
|
|
# SPDX-License-Identifier: MIT
|
|
# Copyright (c) 2026 Rug Munch Media LLC
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import difflib
|
|
import json
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
|
|
import httpx
|
|
from fastapi import APIRouter, Body
|
|
|
|
from client import get_client
|
|
from deps import advanced, scraper
|
|
from errors import ScrapeError
|
|
from settings import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter(tags=["Analysis"])
|
|
|
|
@router.post("/v1/compare", tags=["Analysis"], summary="Compare content of two URLs")
|
|
async def compare(url1: str = Body(...), url2: str = Body(...)) -> dict[str, Any]:
|
|
"""Scrape two URLs and compare their content. Shows additions, deletions, changes."""
|
|
r1, r2 = await asyncio.gather(scraper.scrape(url1), scraper.scrape(url2))
|
|
c1, c2 = r1.get("content", ""), r2.get("content", "")
|
|
diff = list(
|
|
difflib.unified_diff(
|
|
c1.splitlines(keepends=True),
|
|
c2.splitlines(keepends=True),
|
|
fromfile=url1,
|
|
tofile=url2,
|
|
n=3,
|
|
)
|
|
)
|
|
return {
|
|
"success": True,
|
|
"data": {
|
|
"url1": url1,
|
|
"url2": url2,
|
|
"diff": diff[:200],
|
|
"changes": len(diff),
|
|
"identical": len(diff) == 0,
|
|
},
|
|
}
|
|
|
|
|
|
@router.post("/v1/watch", tags=["Analysis"], summary="Watch a page for changes")
|
|
async def watch_page(
|
|
url: str = Body(...), webhook: str = Body(...), interval: int = 3600, selector: str = ""
|
|
) -> dict[str, Any]:
|
|
"""Watch a page for changes. Accepts a webhook URL for notification.
|
|
First call registers the watch, subsequent calls compare and notify.
|
|
Firecrawl charges $99/mo for this feature."""
|
|
result = await scraper.scrape(url, {"bypass_cloudflare": True})
|
|
if result.get("status") != "ok":
|
|
raise ScrapeError("Could not scrape URL")
|
|
content = result.get("content", "")
|
|
diff_result = await advanced.track_diff(url, content)
|
|
# Fire webhook asynchronously on first registration (webhook support)
|
|
if webhook and diff_result["status"] == "registered":
|
|
asyncio.create_task(_fire_watch_webhook(webhook, url, diff_result))
|
|
return {
|
|
"success": True,
|
|
"data": {
|
|
"url": url,
|
|
"status": diff_result["status"],
|
|
"version": diff_result["version"],
|
|
"webhook": webhook,
|
|
"interval": interval,
|
|
"message": "Page registered for change tracking.",
|
|
},
|
|
}
|
|
|
|
|
|
@router.post("/v1/summarize", tags=["Analysis"], summary="AI summarize scraped content")
|
|
async def summarize(url: str = Body(...), max_words: int = 100) -> dict[str, Any]:
|
|
"""Scrape + AI summarize using local Ollama. Free, private."""
|
|
result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30})
|
|
if result.get("status") != "ok":
|
|
raise ScrapeError(result.get("error", "Scrape failed"))
|
|
summary = await advanced.summarize(result.get("content", ""), max_words)
|
|
return {"success": True, "data": {"url": url, **summary}}
|
|
|
|
|
|
@router.post("/v1/diff", tags=["Analysis"], summary="Track page changes over time")
|
|
async def diff(url: str = Body(...), content: str | None = Body(None)) -> dict[str, Any]:
|
|
"""Track page changes over time. Returns diff from last scrape."""
|
|
if content:
|
|
diff_result = await advanced.track_diff(url, content)
|
|
else:
|
|
result = await scraper.scrape(url)
|
|
if result.get("status") != "ok":
|
|
raise ScrapeError("Could not scrape URL")
|
|
diff_result = await advanced.track_diff(url, result.get("content", ""))
|
|
return {"success": True, "data": diff_result}
|
|
|
|
|
|
@router.post("/v1/categorize", tags=["Analysis"], summary="AI-categorize scraped content")
|
|
async def categorize(url: str = Body(...)) -> dict[str, Any]:
|
|
"""AI-categorize scraped content into topic tags."""
|
|
result = await scraper.scrape(url)
|
|
if result.get("status") != "ok":
|
|
raise ScrapeError(result.get("error") or "Categorize failed")
|
|
tags = await advanced.categorize(result.get("content", ""))
|
|
kw = advanced.keyword_density(result.get("content", ""))
|
|
readability = advanced.readability(result.get("content", ""))
|
|
return {
|
|
"success": True,
|
|
"data": {"url": url, "tags": tags, "keywords": kw[:10], "readability": readability},
|
|
}
|
|
|
|
|
|
@router.post("/v1/suggest", tags=["Analysis"], summary="AI-suggest schema fields for a URL")
|
|
async def suggest_schema(data: dict[str, Any] = Body(...)) -> dict[str, Any]:
|
|
url = data.get("url", "")
|
|
|
|
result = await scraper.scrape(url, {"bypass_cloudflare": True, "timeout": 30})
|
|
content = result.get("content", "")
|
|
html = ""
|
|
try:
|
|
client = await get_client()
|
|
resp = await client.get(url, headers={"User-Agent": "Pry/3.0"}, timeout=15)
|
|
html = resp.text
|
|
except httpx.HTTPError:
|
|
logger.warning("suggest_schema_fetch_failed", extra={"url": url})
|
|
schema = {"_page_title": result.get("title", "")}
|
|
candidates: dict[str, str] = {}
|
|
for pattern, field in [
|
|
(r'["\']((?:price|cost|amount)[^"\']*)["\']', "price"),
|
|
(r'["\'](product-title|product-name|item-name)[^"\']*["\']', "name"),
|
|
(r'["\'](?:product-image|main-image|featured-image)[^"\']*["\']', "image"),
|
|
(r'["\'](?:description|product-desc|item-desc)[^"\']*["\']', "description"),
|
|
(r'["\'](?:rating|stars|review-score)[^"\']*["\']', "rating"),
|
|
(r'["\'](?:stock|availability|status)[^"\']*["\']', "stock"),
|
|
]:
|
|
found = re.findall(pattern, html.lower())
|
|
if found:
|
|
candidates[field] = f'[class*="{found[0][:15]}"]'
|
|
if "name" not in candidates:
|
|
h1 = re.findall(r'<h1[^>]*class=["\']([^"\']*)["\']', html)
|
|
if h1:
|
|
candidates["name"] = f"h1.{h1[0].replace(' ', '.')}"
|
|
schema["suggested"] = candidates
|
|
if content and len(content) > 200:
|
|
try:
|
|
prompt = (
|
|
"Analyze this page. Return ONLY JSON with field names as keys and CSS selectors as values. Look for: price, name, description, image, rating, stock.\n\n"
|
|
+ content[:4000]
|
|
)
|
|
client = await get_client()
|
|
ollama_url = settings.ollama_url
|
|
resp = await client.post(
|
|
f"{ollama_url}/api/generate",
|
|
json={
|
|
"model": "qwen2.5-coder:3b",
|
|
"prompt": prompt,
|
|
"stream": False,
|
|
"options": {"num_ctx": 8192, "temperature": 0.1},
|
|
},
|
|
timeout=30,
|
|
)
|
|
if resp.status_code == 200:
|
|
llm_raw = resp.json().get("response", "")
|
|
llm_match = re.search(r"\{.*\}", llm_raw, re.S)
|
|
if llm_match:
|
|
try:
|
|
llm = json.loads(llm_match.group(0))
|
|
if isinstance(llm, dict):
|
|
schema["llm_suggested"] = llm
|
|
except json.JSONDecodeError:
|
|
logger.warning("llm_schema_parse_failed")
|
|
except httpx.HTTPError:
|
|
logger.warning("ollama_suggest_schema_failed")
|
|
return {"success": True, "data": schema}
|
|
|
|
|
|
async def _fire_watch_webhook(webhook: str, url: str, diff_result: dict[str, Any]) -> None:
|
|
"""Fire a webhook notification for watch events."""
|
|
try:
|
|
client = await get_client()
|
|
await client.post(
|
|
webhook,
|
|
json={"event": "watch_update", "url": url, "data": diff_result},
|
|
timeout=10,
|
|
)
|
|
except (httpx.HTTPError, httpx.RequestError):
|
|
logger.exception("watch_webhook_failed", extra={"url": url, "webhook": webhook})
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|