chore(lint): auto-fix 253 of 283 ruff issues (F401, I001, E402, RUF100, UP037, SIM105)
Some checks failed
CI / lint (push) Failing after 2s
CI / typecheck (push) Failing after 2s
CI / test (push) Failing after 2s
CI / Secret scan (gitleaks) (push) Failing after 1s
CI / Security audit (bandit) (push) Failing after 2s

Mass ruff auto-fix:
  - ruff check --fix: 109 issues fixed (F401 unused imports,
    I001 unsorted imports, UP037 quoted annotations, SIM105
    suppressible exception, RUF100 unused-noqa)
  - ruff check --fix --unsafe-fixes: 22 additional issues
  - ruff format: 70 files reformatted
  - Manual pass: fix 16 misplaced import httpx lines
  - Manual pass: fix remaining E402 (import-after-docstring)

Result: 283 errors -> 30 errors.

The remaining 30 are real issues that need manual review:
  5 F401 unused-import (likely auto-generated stubs)
  5 F821 undefined-name (real bugs in code that references
    redis/pydantic/LLMRegistry without imports)
  3 BLE001 (the compliance LLM fallback is intentional; the
    other two are real)
  3 RUF012 mutable-class-default
  3 SIM105, 3 SIM117, 2 E722, 2 E741
  1 B007, 1 B025, 1 E402, 1 RUF200 (pyproject.toml issue)

Tests: 436/437 pass (1 pre-existing SSE sandbox failure).
format check + import sort: now clean.
make ci: still gated on the 30 remaining real issues.
Follow-up: triage the 30 issues file-by-file.
This commit is contained in:
Crypto Rug Munch 2026-07-02 21:51:25 +02:00
parent e60a62a07a
commit a7c30b12cd
85 changed files with 2374 additions and 1071 deletions

197
db.py
View file

@ -50,6 +50,7 @@ the corresponding JSON store is migrated.
Part of Pry - https://git.rugmunch.io/RugMunchMedia/pryscraper
Licensed under MIT. See LICENSE.
"""
# SPDX-License-Identifier: MIT
# Copyright (c) 2026 Rug Munch Media LLC
#
@ -61,10 +62,11 @@ import json
import logging
import os
import sqlite3
from collections.abc import Iterator
from contextlib import contextmanager
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Iterator
from typing import Any
try:
from sqlalchemy import (
@ -88,7 +90,7 @@ try:
except ImportError: # pragma: no cover
_HAS_SA = False
from paths import PRY_DATA_DIR # noqa: E402
from paths import PRY_DATA_DIR
logger = logging.getLogger(__name__)
@ -98,10 +100,11 @@ DEFAULT_DB_PATH = PRY_DATA_DIR / "pry.db"
_has_sqlalchemy = _HAS_SA
def get_db() -> "Engine":
def get_db() -> Engine:
"""Backward-compat alias for get_engine()."""
return get_engine()
DATABASE_URL_ENV = "PRY_DATABASE_URL"
DEFAULT_SQLITE_URL = f"sqlite:///{DEFAULT_DB_PATH}"
@ -114,11 +117,11 @@ def _resolve_database_url() -> str:
return DEFAULT_SQLITE_URL
_engine: "Engine | None" = None
_SessionLocal: "sessionmaker[Session] | None" = None
_engine: Engine | None = None
_SessionLocal: sessionmaker[Session] | None = None
def get_engine() -> "Engine":
def get_engine() -> Engine:
"""Get the SQLAlchemy engine. Lazily creates one on first call."""
global _engine, _SessionLocal
if not _HAS_SA:
@ -131,19 +134,23 @@ def get_engine() -> "Engine":
_engine = create_engine(url, connect_args=connect_args, future=True, echo=False)
# Enable foreign keys for SQLite (off by default)
if url.startswith("sqlite"):
@event.listens_for(_engine, "connect")
def _enable_sqlite_fk(dbapi_conn: sqlite3.Connection, _conn_record: Any) -> None:
cur = dbapi_conn.cursor()
cur.execute("PRAGMA foreign_keys=ON")
cur.close()
_SessionLocal = sessionmaker(bind=_engine, autoflush=False, expire_on_commit=False)
# Auto-create tables on first import (idempotent)
Base.metadata.create_all(_engine)
logger.info("db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url})
logger.info(
"db_engine_initialized", extra={"url": url.split("@")[-1] if "@" in url else url}
)
return _engine
def get_session() -> "Session":
def get_session() -> Session:
"""Get a new Session. Caller is responsible for closing it.
Use `with db_session() as s:` or the `session_scope()` context manager
@ -156,7 +163,7 @@ def get_session() -> "Session":
@contextmanager
def session_scope() -> Iterator["Session"]:
def session_scope() -> Iterator[Session]:
"""Context manager: yields a Session, commits on success, rolls back on error.
Usage:
@ -178,6 +185,7 @@ def session_scope() -> Iterator["Session"]:
# ── Model base ────────────────────────────────────────────────────
if _HAS_SA:
class Base(DeclarativeBase):
pass
@ -282,7 +290,9 @@ if _HAS_SA:
class MonitorRun(Base):
__tablename__ = "monitor_runs"
id = Column(Integer, primary_key=True)
monitor_id = Column(String(64), ForeignKey("monitors.monitor_id"), index=True, nullable=False)
monitor_id = Column(
String(64), ForeignKey("monitors.monitor_id"), index=True, nullable=False
)
ts = Column(DateTime, default=_now, index=True)
status = Column(String(16), default="success") # success|changed|error
diff = Column(JSON, default=dict)
@ -350,7 +360,9 @@ if _HAS_SA:
class PipelineRun(Base):
__tablename__ = "pipeline_runs"
id = Column(Integer, primary_key=True)
pipeline_id = Column(String(64), ForeignKey("pipelines.pipeline_id"), index=True, nullable=False)
pipeline_id = Column(
String(64), ForeignKey("pipelines.pipeline_id"), index=True, nullable=False
)
ts = Column(DateTime, default=_now, index=True)
status = Column(String(16), default="success")
result = Column(JSON, default=dict)
@ -477,6 +489,7 @@ else: # pragma: no cover
# ── Importer: JSON store -> SQL ───────────────────────────────────
def _iter_jsonl(path: Path) -> Iterator[dict[str, Any]]:
if not path.exists():
return
@ -526,29 +539,33 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
# Quality
n = 0
for rec in _iter_jsonl(data_dir / "quality" / "checks.jsonl"):
s.add(QualityCheck(
extraction_id=rec.get("extraction_id", rec.get("id", ""))[:64],
url=rec.get("url", ""),
completeness=rec.get("completeness", 0.0),
accuracy=rec.get("accuracy", 0.0),
freshness=rec.get("freshness", 0.0),
overall_score=rec.get("overall_score", 0.0),
risk_level=rec.get("risk_level", "unknown"),
issues=rec.get("issues", []),
data=rec,
))
s.add(
QualityCheck(
extraction_id=rec.get("extraction_id", rec.get("id", ""))[:64],
url=rec.get("url", ""),
completeness=rec.get("completeness", 0.0),
accuracy=rec.get("accuracy", 0.0),
freshness=rec.get("freshness", 0.0),
overall_score=rec.get("overall_score", 0.0),
risk_level=rec.get("risk_level", "unknown"),
issues=rec.get("issues", []),
data=rec,
)
)
n += 1
counts["quality"] = n
# Intel (JSONL)
n = 0
for rec in _iter_jsonl(data_dir / "intel" / "snapshots.jsonl"):
s.add(IntelSnapshot(
competitor_id=rec.get("competitor_id", "")[:64],
competitor_name=rec.get("competitor_name", "")[:256],
url=rec.get("url", ""),
fields=rec.get("fields", {}),
))
s.add(
IntelSnapshot(
competitor_id=rec.get("competitor_id", "")[:64],
competitor_name=rec.get("competitor_name", "")[:256],
url=rec.get("url", ""),
fields=rec.get("fields", {}),
)
)
n += 1
counts["intel"] = n
@ -556,15 +573,17 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
n = 0
for rec in _iter_json_files(data_dir / "monitors"):
mid = rec.get("monitor_id") or rec.get("id", "")
s.add(Monitor(
monitor_id=mid[:64],
url=rec.get("url", ""),
monitor_name=rec.get("name", ""), # was 'name' in JSON
schedule_cron=rec.get("schedule_cron", ""),
check_type=rec.get("check_type", "content"),
active=rec.get("active", True),
webhook_url=rec.get("webhook_url", ""),
))
s.add(
Monitor(
monitor_id=mid[:64],
url=rec.get("url", ""),
monitor_name=rec.get("name", ""), # was 'name' in JSON
schedule_cron=rec.get("schedule_cron", ""),
check_type=rec.get("check_type", "content"),
active=rec.get("active", True),
webhook_url=rec.get("webhook_url", ""),
)
)
n += 1
counts["monitors"] = n
@ -572,14 +591,16 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
n = 0
for rec in _iter_json_files(data_dir / "accounts"):
aid = rec.get("account_id") or rec.get("id", "")
s.add(Account(
account_id=aid[:64],
service=rec.get("service", "unknown")[:64],
username=rec.get("username", ""),
email=rec.get("email", ""),
status=rec.get("status", "active"),
used_count=rec.get("used_count", 0),
))
s.add(
Account(
account_id=aid[:64],
service=rec.get("service", "unknown")[:64],
username=rec.get("username", ""),
email=rec.get("email", ""),
status=rec.get("status", "active"),
used_count=rec.get("used_count", 0),
)
)
n += 1
counts["accounts"] = n
@ -587,20 +608,22 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
n = 0
for rec in _iter_json_files(data_dir / "actors"):
aid = rec.get("actor_id") or rec.get("id", "")
s.add(Actor(
actor_id=aid[:64],
name=rec.get("name", ""),
description=rec.get("description", ""),
template_id=rec.get("template_id", ""),
code=rec.get("code", ""),
price_per_run=rec.get("price_per_run", 0.0),
visibility=rec.get("visibility", "private"),
schedule_cron=rec.get("schedule_cron", ""),
tags=rec.get("tags", []),
author=rec.get("author", ""),
run_count=rec.get("run_count", 0),
revenue_usd=rec.get("revenue_usd", 0.0),
))
s.add(
Actor(
actor_id=aid[:64],
name=rec.get("name", ""),
description=rec.get("description", ""),
template_id=rec.get("template_id", ""),
code=rec.get("code", ""),
price_per_run=rec.get("price_per_run", 0.0),
visibility=rec.get("visibility", "private"),
schedule_cron=rec.get("schedule_cron", ""),
tags=rec.get("tags", []),
author=rec.get("author", ""),
run_count=rec.get("run_count", 0),
revenue_usd=rec.get("revenue_usd", 0.0),
)
)
n += 1
counts["actors"] = n
@ -608,13 +631,15 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
n = 0
for rec in _iter_json_files(data_dir / "pipelines"):
pid = rec.get("pipeline_id") or rec.get("id", "")
s.add(Pipeline(
pipeline_id=pid[:64],
pipeline_name=rec.get("name", ""),
steps=rec.get("steps", []),
schedule_cron=rec.get("schedule_cron", ""),
active=rec.get("active", True),
))
s.add(
Pipeline(
pipeline_id=pid[:64],
pipeline_name=rec.get("name", ""),
steps=rec.get("steps", []),
schedule_cron=rec.get("schedule_cron", ""),
active=rec.get("active", True),
)
)
n += 1
counts["pipelines"] = n
@ -622,13 +647,15 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
n = 0
for rec in _iter_json_files(data_dir / "sessions"):
sid = rec.get("session_id") or rec.get("id", "")
s.add(BrowserSession(
session_id=sid[:64],
cookies=rec.get("cookies", []),
local_storage=rec.get("local_storage", {}),
user_agent=rec.get("user_agent", ""),
proxy=rec.get("proxy", ""),
))
s.add(
BrowserSession(
session_id=sid[:64],
cookies=rec.get("cookies", []),
local_storage=rec.get("local_storage", {}),
user_agent=rec.get("user_agent", ""),
proxy=rec.get("proxy", ""),
)
)
n += 1
counts["sessions"] = n
@ -636,15 +663,17 @@ def import_json_stores(data_dir: Path | None = None) -> dict[str, int]:
n = 0
for rec in _iter_json_files(data_dir / "agency"):
aid = rec.get("agency_id") or rec.get("id", "")
s.add(Agency(
agency_id=aid[:64],
agency_name=rec.get("name", ""),
owner_email=rec.get("owner_email", ""),
custom_domain=rec.get("custom_domain", ""),
brand_color=rec.get("brand_color", "#f59e0b"),
logo_url=rec.get("logo_url", ""),
quota=rec.get("quota", 10000),
))
s.add(
Agency(
agency_id=aid[:64],
agency_name=rec.get("name", ""),
owner_email=rec.get("owner_email", ""),
custom_domain=rec.get("custom_domain", ""),
brand_color=rec.get("brand_color", "#f59e0b"),
logo_url=rec.get("logo_url", ""),
quota=rec.get("quota", 10000),
)
)
n += 1
counts["agency"] = n
@ -666,7 +695,9 @@ def db_health() -> dict[str, Any]:
sqlite_version = None
return {
"available": True,
"url": _resolve_database_url().split("@")[-1] if "@" in _resolve_database_url() else _resolve_database_url(),
"url": _resolve_database_url().split("@")[-1]
if "@" in _resolve_database_url()
else _resolve_database_url(),
"sqlite_version": sqlite_version,
}
except Exception as e: