diff --git a/api.py b/api.py index 7de7ffa..323cf94 100644 --- a/api.py +++ b/api.py @@ -26,6 +26,8 @@ from typing import Any, cast from urllib.parse import urljoin, urlparse import httpx +import pydantic +import redis import uvicorn from fastapi import Body, FastAPI, Request, Response, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware @@ -53,6 +55,7 @@ from parser import DocumentParser from pipeline import HOOK_POINTS, get_pipeline, run_pipeline from pryextras import BatchProcessor, TransformEngine, recorder, streams from ratelimit import RateLimiter +from routers.auth import router as auth_router from scraper import BlockDetector, PryScraper from settings import settings from x402_middleware import X402Middleware @@ -3135,7 +3138,6 @@ async def list_schemas() -> dict[str, Any]: # ── Auth ── # (split into routers/auth.py on the api-router-split refactor) -from routers.auth import router as auth_router app.include_router(auth_router) diff --git a/automator.py b/automator.py index 22d9ab2..7f7040b 100644 --- a/automator.py +++ b/automator.py @@ -73,10 +73,8 @@ class PryAutomator: """Periodically clean up stale sessions to prevent memory leaks.""" while True: await asyncio.sleep(SESSION_CLEANUP_INTERVAL) - try: + with contextlib.suppress(Exception): await self._cleanup_stale_sessions() - except Exception: # noqa: BLE001 - pass async def _get_or_create_session( self, session_id: str | None = None, headless: bool = True, viewport: dict | None = None @@ -281,10 +279,8 @@ class PryAutomator: ) -> str: session = await self._get_or_create_session() if cookies: - try: + with contextlib.suppress(Exception): await session.context.add_cookies(cookies) - except Exception: # noqa: BLE001 - pass await session.page.goto(url, wait_until="networkidle") return session.id diff --git a/camoufox_integration.py b/camoufox_integration.py index 6b15dfc..df2b7ce 100644 --- a/camoufox_integration.py +++ b/camoufox_integration.py @@ -10,17 +10,17 @@ for Playwright that focuses on stealth.""" # Licensed under Business Source License 1.1 — see LICENSE-BSL-STEALTH. # Change Date: 2029-01-01 (converts to MIT). +import contextlib import logging import random import time -from typing import Any +from typing import Any, ClassVar logger = logging.getLogger(__name__) # Check for camoufox try: from camoufox import AsyncCamoufox - from camoufox.utils import DefaultAddons _has_camoufox = True except ImportError: @@ -30,7 +30,7 @@ except ImportError: class CamoufoxBrowser: """Anti-detection Firefox browser using Camoufox.""" - DEFAULT_CONFIGS = { + DEFAULT_CONFIGS: ClassVar[dict[str, dict[str, Any]]] = { "chrome_windows": { "headless": True, "os": "windows", @@ -108,10 +108,8 @@ class CamoufoxBrowser: await page.context.add_cookies([cookie]) await page.goto(url, wait_until="domcontentloaded") if wait_selector: - try: + with contextlib.suppress(Exception): await page.wait_for_selector(wait_selector, timeout=wait_time) - except Exception: # noqa: BLE001 - pass else: await page.wait_for_timeout(wait_time) content = await page.content() diff --git a/db.py b/db.py index 9300412..2f8325c 100644 --- a/db.py +++ b/db.py @@ -84,6 +84,7 @@ try: event, ) from sqlalchemy.engine import Engine + from sqlalchemy.exc import OperationalError, ProgrammingError, SQLAlchemyError from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker _HAS_SA = True @@ -691,7 +692,7 @@ def db_health() -> dict[str, Any]: try: result = conn.exec_driver_sql("SELECT sqlite_version()").fetchone() sqlite_version = result[0] if result else None - except Exception: + except (OperationalError, ProgrammingError): sqlite_version = None return { "available": True, @@ -700,5 +701,5 @@ def db_health() -> dict[str, Any]: else _resolve_database_url(), "sqlite_version": sqlite_version, } - except Exception as e: + except SQLAlchemyError as e: return {"available": False, "error": str(e)[:200]} diff --git a/destinations.py b/destinations.py index be48301..1519083 100644 --- a/destinations.py +++ b/destinations.py @@ -144,7 +144,7 @@ async def _write_to_sheets_api( except json.JSONDecodeError: return {"success": False, "error": "Invalid credentials JSON"} - except (json.JSONDecodeError, ValueError) as e: + except ValueError as e: return {"success": False, "error": str(e)[:200]} diff --git a/jobqueue.py b/jobqueue.py index dfd0fa2..8494269 100644 --- a/jobqueue.py +++ b/jobqueue.py @@ -39,7 +39,7 @@ class JobQueue: try: self._redis = await aioredis.from_url(self.redis_url, socket_timeout=3) await self._redis.ping() - except: + except (aioredis.RedisError, OSError, TimeoutError): self._redis = None # Fallback to local storage return self._redis @@ -121,5 +121,5 @@ class JobQueue: "Content-Type": "application/json", }, ) - except: + except (httpx.RequestError, OSError): pass # Webhook fire is best-effort diff --git a/llm_providers/providers.py b/llm_providers/providers.py index 7944250..0067a9b 100644 --- a/llm_providers/providers.py +++ b/llm_providers/providers.py @@ -5,10 +5,13 @@ # Licensed under MIT. See LICENSE. """Concrete LLM provider implementations.""" +from __future__ import annotations + import logging import os from llm_providers.base import LLMProvider, LLMResponse +from llm_providers.registry import LLMRegistry logger = logging.getLogger(__name__) @@ -348,7 +351,7 @@ class OpenRouterProvider(LLMProvider): raise NotImplementedError("OpenRouter focuses on chat; use dedicated embedding providers") -def register_default_providers(registry: "LLMRegistry") -> None: +def register_default_providers(registry: LLMRegistry) -> None: """Register all providers whose API keys are set in environment.""" api_key_map = { "openai": os.getenv("OPENAI_API_KEY"), diff --git a/logging_config.py b/logging_config.py index 100a4cd..aec7dc5 100644 --- a/logging_config.py +++ b/logging_config.py @@ -147,10 +147,6 @@ def _add_service_name(_, __, event_dict: dict[str, Any]) -> dict[str, Any]: def _add_request_id_if_present(_, __, event_dict: dict[str, Any]) -> dict[str, Any]: """Pull request_id from contextvars (set by middleware) if present.""" - try: - import contextvars - except ImportError: - return event_dict # The middleware (or FastAPI dependency) sets this. We don't import # the middleware here to avoid circular imports. return event_dict diff --git a/pyproject.toml b/pyproject.toml index ebae998..4400d79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,8 +27,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed", ] -[project.license-files] -paths = ["LICENSE", "LICENSE-BSL-STEALTH"] +license-files = ["LICENSE", "LICENSE-BSL-STEALTH"] dependencies = [ "fastapi>=0.115.0", "uvicorn[standard]>=0.32.0", diff --git a/reconciliation.py b/reconciliation.py index adb2a40..1d7629b 100644 --- a/reconciliation.py +++ b/reconciliation.py @@ -388,7 +388,7 @@ async def llm_enhance_reconciliation( groups.setdefault(gid, []).append(e) verified = 0 refuted = 0 - for gid, group in groups.items(): + for _gid, group in groups.items(): result = await llm_entity_reconcile(group, vertical="product") if result.get("is_same_entity"): verified += 1 diff --git a/scraper.py b/scraper.py index 5273a3e..2872335 100644 --- a/scraper.py +++ b/scraper.py @@ -12,7 +12,7 @@ Beats every anti-bot system: Cloudflare, DataDome, Akamai, PerimeterX.""" import asyncio import random import re -from typing import Any +from typing import Any, ClassVar from urllib.parse import urljoin, urlparse import httpx @@ -55,7 +55,7 @@ CLOUDFLARE_INDICATORS = [ class BlockDetector: """Detect anti-bot blocks by analyzing HTML, status codes, and headers.""" - VENDOR_PATTERNS = { + VENDOR_PATTERNS: ClassVar[dict[str, list[str]]] = { "cloudflare": [ "cloudflare", "cf-", @@ -70,7 +70,7 @@ class BlockDetector: "imperva": ["imperva", "incapsula"], } - GENERIC_PATTERNS = [ + GENERIC_PATTERNS: ClassVar[list[str]] = [ "captcha", "access denied", "blocked", @@ -533,7 +533,7 @@ class PryScraper: links = await self._extract_links( current_url, {"limit": max_pages - len(visited)} ) - to_visit.extend((l, depth + 1) for l in links if l not in visited) + to_visit.extend((link, depth + 1) for link in links if link not in visited) except Exception: # noqa: BLE001 continue return results diff --git a/tasks.py b/tasks.py index ab1f5f1..1cd98d4 100644 --- a/tasks.py +++ b/tasks.py @@ -7,6 +7,7 @@ Long-running jobs like crawls, monitors, and bulk imports should not block reque # Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper # Licensed under MIT. See LICENSE. import asyncio +import importlib.util import json import logging import uuid @@ -20,12 +21,7 @@ from paths import PRY_DATA_DIR logger = logging.getLogger(__name__) # Try Celery -try: - from celery import Celery - - _has_celery = True -except ImportError: - _has_celery = False +_has_celery = importlib.util.find_spec("celery") is not None JOBS_DIR = PRY_DATA_DIR / "jobs" JOBS_DIR.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_api_mcp_x402.py b/tests/test_api_mcp_x402.py index 5e04eda..68a837b 100644 --- a/tests/test_api_mcp_x402.py +++ b/tests/test_api_mcp_x402.py @@ -12,12 +12,15 @@ import json import socket import subprocess import time +from pathlib import Path import httpx from fastapi.testclient import TestClient from api import app +REPO_ROOT = Path(__file__).resolve().parents[1] + def _find_free_port() -> int: """Return an available TCP port on 127.0.0.1.""" @@ -31,9 +34,9 @@ def _wait_for_server(url: str, timeout: float = 30.0) -> None: deadline = time.time() + timeout while time.time() < deadline: try: - httpx.get(url, timeout=1.0) + httpx.get(url, timeout=5.0) return - except Exception: + except httpx.RequestError: time.sleep(0.5) raise TimeoutError(f"Server at {url} did not start within {timeout}s") @@ -194,7 +197,7 @@ class TestMCPSSE: "--log-level", "warning", ], - cwd="/home/dev/pry", + cwd=str(REPO_ROOT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) diff --git a/tests/test_llm_fallback.py b/tests/test_llm_fallback.py index 13a2adc..1be1f9e 100644 --- a/tests/test_llm_fallback.py +++ b/tests/test_llm_fallback.py @@ -101,19 +101,19 @@ async def test_seo_llm_enhancement_fills_empty_critical_fields(): patch("seo_monitor._get_attr", return_value=""), patch("seo_monitor._count_links", return_value=0), patch("llm_features.llm_seo_analyze", fake_llm), + patch("client.get_client") as mock_get_client, ): # Mock the HTTP client to return a fake HTML response - with patch("client.get_client") as mock_get_client: - mock_resp = MagicMock() - mock_resp.is_success = True - mock_resp.text = "