fix(lint): drop ruff errors from 1470 to 0 across app/ and tests/
- Exclude generated SDK (sdks/python) and operational scripts from ruff lint - Add targeted per-file ignores for ASYNC*, S310, S603, S607, S108, S314, S102, PIE810, SIM102 in scripts/ - Auto-fix safe categories (I001, F401, W292, F841, PIE790, RUF100, etc.) - Bulk-fix S110 (try-except-pass), S112 (try-except-continue), S311 (random), S324 (md5/sha1), S301 (pickle) and similar lint categories - Rename N806 non-lowercase locals, including ML X/y variables preserved with noqa for scikit-learn conventions - Replace urllib.request calls with httpx.AsyncClient / httpx.Client (S310) - Wrap blocking os.path/os calls in asyncio.get_running_loop().run_in_executor - Replace subprocess.run with asyncio.create_subprocess_exec in async contexts - Store asyncio.create_task return values in _background_tasks set (RUF006) - Convert hardcoded subprocess binary names to absolute paths (S607) where appropriate; add noqa where path is config-driven (CAST_PATH, etc.) - Parameterize SQL queries with placeholders and add noqa for sanitized inputs - Fix all mechanical categories: SIM102, PIE810, TC001/2/3, S108, S314, S107, S306, S301, N802/N815/N817, S104, S605, S501, RUF022, UP031 - Add missing 'import asyncio' where referenced but not imported (F821) - Fix E402 module-import-not-at-top by adding '# noqa: E402' for circular-import safe cases and code-defined imports - Remove hardcoded Redis password in databus_warm_cron.py; use env vars Tests: - Add tests/unit/core/test_ai_router.py (8 tests): model resolution, chat completion with mocked httpx, fallback to OpenRouter, no-provider error, streaming - Add tests/unit/core/test_tracing.py (7 tests): setup_otel disabled/enabled, shutdown_otel, span helpers, tracing-enabled route registration - Add tests/unit/core/test_langfuse.py (2 tests): no-env init, noop flush - Fix tests/unit/domain/scanner/test_service.py to import from the moved app.domains.scanners.core.service Result: 'ruff check .' passes with 0 errors (was 1470). Pytest: 808 passed, 1 skipped (no regressions).
This commit is contained in:
parent
8491635bf2
commit
cfd75fd1a0
290 changed files with 5417 additions and 2458 deletions
|
|
@ -42,7 +42,9 @@ def _get_url():
|
|||
|
||||
|
||||
def _get_key():
|
||||
return os.environ.get("SUPABASE_SERVICE_KEY", "") or os.environ.get("SUPABASE_SERVICE_ROLE_KEY", "")
|
||||
return os.environ.get("SUPABASE_SERVICE_KEY", "") or os.environ.get(
|
||||
"SUPABASE_SERVICE_ROLE_KEY", ""
|
||||
)
|
||||
|
||||
|
||||
def _get_headers():
|
||||
|
|
@ -98,18 +100,24 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
|
|||
|
||||
for attempt in range(MAX_RETRIES):
|
||||
try:
|
||||
resp = await client.post(url, json=rows, params=params, headers=_get_headers(), timeout=60)
|
||||
resp = await client.post(
|
||||
url, json=rows, params=params, headers=_get_headers(), timeout=60
|
||||
)
|
||||
if resp.status_code in (200, 201):
|
||||
return len(rows)
|
||||
elif resp.status_code == 500 and "timeout" in (resp.text or "").lower():
|
||||
# Statement timeout - split into smaller batches
|
||||
logger.warning(f"Batch {batch_num}: statement timeout (attempt {attempt + 1}), splitting")
|
||||
logger.warning(
|
||||
f"Batch {batch_num}: statement timeout (attempt {attempt + 1}), splitting"
|
||||
)
|
||||
if len(rows) > 10:
|
||||
# Split into two halves
|
||||
mid = len(rows) // 2
|
||||
count = 0
|
||||
for half in [rows[:mid], rows[mid:]]:
|
||||
r2 = await client.post(url, json=half, params=params, headers=_get_headers(), timeout=90)
|
||||
r2 = await client.post(
|
||||
url, json=half, params=params, headers=_get_headers(), timeout=90
|
||||
)
|
||||
if r2.status_code in (200, 201):
|
||||
count += len(half)
|
||||
else:
|
||||
|
|
@ -127,10 +135,14 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
|
|||
count += 1
|
||||
await asyncio.sleep(0.02)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning(
|
||||
"swallowed exception", exc_info=True
|
||||
)
|
||||
return count
|
||||
else:
|
||||
logger.error(f"Batch {batch_num} insert failed: {resp.status_code} {resp.text[:300]}")
|
||||
logger.error(
|
||||
f"Batch {batch_num} insert failed: {resp.status_code} {resp.text[:300]}"
|
||||
)
|
||||
return 0
|
||||
except httpx.ReadTimeout:
|
||||
logger.warning(f"Batch {batch_num}: timeout (attempt {attempt + 1})")
|
||||
|
|
@ -142,7 +154,9 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
|
|||
return 0
|
||||
|
||||
|
||||
async def migrate_collection(r: aioredis.Redis, client: httpx.AsyncClient, collection: str, existing_ids: set) -> int:
|
||||
async def migrate_collection(
|
||||
r: aioredis.Redis, client: httpx.AsyncClient, collection: str, existing_ids: set
|
||||
) -> int:
|
||||
"""Migrate all docs from one Redis collection to Supabase."""
|
||||
idx_key = f"rag:idx:{collection}"
|
||||
doc_ids = await r.smembers(idx_key)
|
||||
|
|
@ -198,7 +212,9 @@ async def migrate_collection(r: aioredis.Redis, client: httpx.AsyncClient, colle
|
|||
migrated += count
|
||||
batch_count += 1
|
||||
if batch_count % 10 == 0:
|
||||
logger.info(f" Progress: {migrated}/{len(to_migrate)} migrated ({batch_count} batches)")
|
||||
logger.info(
|
||||
f" Progress: {migrated}/{len(to_migrate)} migrated ({batch_count} batches)"
|
||||
)
|
||||
batch_rows = []
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue