chore(lint): auto-fix 253 of 283 ruff issues (F401, I001, E402, RUF100, UP037, SIM105)
Mass ruff auto-fix:
- ruff check --fix: 109 issues fixed (F401 unused imports,
I001 unsorted imports, UP037 quoted annotations, SIM105
suppressible exception, RUF100 unused-noqa)
- ruff check --fix --unsafe-fixes: 22 additional issues
- ruff format: 70 files reformatted
- Manual pass: fix 16 misplaced import httpx lines
- Manual pass: fix remaining E402 (import-after-docstring)
Result: 283 errors -> 30 errors.
The remaining 30 are real issues that need manual review:
5 F401 unused-import (likely auto-generated stubs)
5 F821 undefined-name (real bugs in code that references
redis/pydantic/LLMRegistry without imports)
3 BLE001 (the compliance LLM fallback is intentional; the
other two are real)
3 RUF012 mutable-class-default
3 SIM105, 3 SIM117, 2 E722, 2 E741
1 B007, 1 B025, 1 E402, 1 RUF200 (pyproject.toml issue)
Tests: 436/437 pass (1 pre-existing SSE sandbox failure).
format check + import sort: now clean.
make ci: still gated on the 30 remaining real issues.
Follow-up: triage the 30 issues file-by-file.
This commit is contained in:
parent
e60a62a07a
commit
a7c30b12cd
85 changed files with 2374 additions and 1071 deletions
36
tasks.py
36
tasks.py
|
|
@ -1,29 +1,28 @@
|
|||
"""Pry — Background job system using asyncio (built-in) or Celery if available.
|
||||
Long-running jobs like crawls, monitors, and bulk imports should not block request threads."""
|
||||
from paths import PRY_DATA_DIR
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
# Copyright (c) 2026 Rug Munch Media LLC
|
||||
#
|
||||
# Part of Pry — https://git.rugmunch.io/RugMunchMedia/pryscraper
|
||||
# Licensed under MIT. See LICENSE.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from paths import PRY_DATA_DIR
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Try Celery
|
||||
try:
|
||||
from celery import Celery
|
||||
|
||||
_has_celery = True
|
||||
except ImportError:
|
||||
_has_celery = False
|
||||
|
|
@ -32,7 +31,7 @@ JOBS_DIR = PRY_DATA_DIR / "jobs"
|
|||
JOBS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
class JobStatus(StrEnum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
|
|
@ -42,7 +41,10 @@ class JobStatus(str, Enum):
|
|||
|
||||
class Job:
|
||||
"""Represents a background job."""
|
||||
def __init__(self, job_id: str, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None):
|
||||
|
||||
def __init__(
|
||||
self, job_id: str, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None
|
||||
):
|
||||
self.id = job_id
|
||||
self.name = name
|
||||
self.func = func
|
||||
|
|
@ -57,9 +59,14 @@ class Job:
|
|||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id, "name": self.name, "status": self.status.value,
|
||||
"created_at": self.created_at, "started_at": self.started_at,
|
||||
"completed_at": self.completed_at, "result": self.result, "error": self.error,
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"status": self.status.value,
|
||||
"created_at": self.created_at,
|
||||
"started_at": self.started_at,
|
||||
"completed_at": self.completed_at,
|
||||
"result": self.result,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -75,7 +82,9 @@ class JobQueue:
|
|||
self._workers: list[asyncio.Task] = []
|
||||
self._running = False
|
||||
|
||||
async def submit(self, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None) -> str:
|
||||
async def submit(
|
||||
self, name: str, func: Callable, args: tuple = (), kwargs: dict | None = None
|
||||
) -> str:
|
||||
job_id = uuid.uuid4().hex[:12]
|
||||
job = Job(job_id, name, func, args, kwargs)
|
||||
self.jobs[job_id] = job
|
||||
|
|
@ -86,7 +95,8 @@ class JobQueue:
|
|||
return job_id
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._running or self._queue is None: return
|
||||
if self._running or self._queue is None:
|
||||
return
|
||||
self._running = True
|
||||
for i in range(self.max_workers):
|
||||
worker = asyncio.create_task(self._worker(f"worker-{i}"))
|
||||
|
|
@ -109,7 +119,7 @@ class JobQueue:
|
|||
else:
|
||||
job.result = await asyncio.to_thread(job.func, *job.args, **job.kwargs)
|
||||
job.status = JobStatus.COMPLETED
|
||||
except Exception as e: # noqa: BLE001
|
||||
except Exception as e:
|
||||
job.error = str(e)[:500]
|
||||
job.status = JobStatus.FAILED
|
||||
logger.exception("job_failed", extra={"job_id": job.id, "error": str(e)})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue