fix(lint): drop ruff errors from 1470 to 0 across app/ and tests/

- Exclude generated SDK (sdks/python) and operational scripts from ruff lint
- Add targeted per-file ignores for ASYNC*, S310, S603, S607, S108, S314,
  S102, PIE810, SIM102 in scripts/
- Auto-fix safe categories (I001, F401, W292, F841, PIE790, RUF100, etc.)
- Bulk-fix S110 (try-except-pass), S112 (try-except-continue), S311 (random),
  S324 (md5/sha1), S301 (pickle) and similar lint categories
- Rename N806 non-lowercase locals, including ML X/y variables preserved
  with noqa for scikit-learn conventions
- Replace urllib.request calls with httpx.AsyncClient / httpx.Client (S310)
- Wrap blocking os.path/os calls in asyncio.get_running_loop().run_in_executor
- Replace subprocess.run with asyncio.create_subprocess_exec in async contexts
- Store asyncio.create_task return values in _background_tasks set (RUF006)
- Convert hardcoded subprocess binary names to absolute paths (S607) where
  appropriate; add noqa where path is config-driven (CAST_PATH, etc.)
- Parameterize SQL queries with placeholders and add noqa for sanitized inputs
- Fix all mechanical categories: SIM102, PIE810, TC001/2/3, S108, S314, S107,
  S306, S301, N802/N815/N817, S104, S605, S501, RUF022, UP031
- Add missing 'import asyncio' where referenced but not imported (F821)
- Fix E402 module-import-not-at-top by adding '# noqa: E402' for circular-import
  safe cases and code-defined imports
- Remove hardcoded Redis password in databus_warm_cron.py; use env vars

Tests:
- Add tests/unit/core/test_ai_router.py (8 tests): model resolution, chat
  completion with mocked httpx, fallback to OpenRouter, no-provider error,
  streaming
- Add tests/unit/core/test_tracing.py (7 tests): setup_otel disabled/enabled,
  shutdown_otel, span helpers, tracing-enabled route registration
- Add tests/unit/core/test_langfuse.py (2 tests): no-env init, noop flush
- Fix tests/unit/domain/scanner/test_service.py to import from the moved
  app.domains.scanners.core.service

Result: 'ruff check .' passes with 0 errors (was 1470).
Pytest: 808 passed, 1 skipped (no regressions).
This commit is contained in:
Crypto Rug Munch 2026-07-08 03:36:58 +07:00
parent 8491635bf2
commit cfd75fd1a0
290 changed files with 5417 additions and 2458 deletions

View file

@ -9,6 +9,7 @@ Usage: python airdrop_v2_distribute.py --token NEW_TOKEN --amount TOTAL_AMOUNT -
import argparse
import json
import logging
from dataclasses import dataclass
from pathlib import Path
@ -62,7 +63,7 @@ class AirdropV2Distributor:
if pattern in value_str:
return True, f"Redis label: {key}={value}"
except: # noqa: E722
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
# Check local scam lists
scam_dir = Path("/root/backend/data/scam_lists")
@ -73,7 +74,7 @@ class AirdropV2Distributor:
if address in str(data):
return True, f"Found in {f.name}"
except: # noqa: E722
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
return False, None
@ -96,7 +97,7 @@ class AirdropV2Distributor:
address=addr,
chain="solana",
amount=amount,
original_token="CRM",
original_token="CRM", # noqa: S106
percentage=float(holder.get("Percentage", 0)),
is_blacklisted=is_blacklisted,
blacklist_reason=reason,
@ -119,7 +120,7 @@ class AirdropV2Distributor:
address=addr,
chain="base",
amount=amount,
original_token="$cryptorugmunch",
original_token="$cryptorugmunch", # noqa: S106
percentage=0, # Not provided in base CSV
is_blacklisted=is_blacklisted,
blacklist_reason=reason,
@ -128,7 +129,9 @@ class AirdropV2Distributor:
return qualified
def calculate_distribution(self, total_amount: float, distribution_type: str = "proportional") -> dict[str, float]:
def calculate_distribution(
self, total_amount: float, distribution_type: str = "proportional"
) -> dict[str, float]:
"""Calculate airdrop amounts per holder"""
qualified = [h for h in self.qualify_holders() if not h.is_blacklisted]
@ -158,11 +161,16 @@ class AirdropV2Distributor:
# Normalize to total amount
total_weighted = sum(distribution.values())
return {addr: (amount / total_weighted) * total_amount for addr, amount in distribution.items()}
return {
addr: (amount / total_weighted) * total_amount
for addr, amount in distribution.items()
}
return {}
def generate_distribution_list(self, token_address: str, total_amount: float, chain: str = "solana") -> dict:
def generate_distribution_list(
self, token_address: str, total_amount: float, chain: str = "solana"
) -> dict:
"""Generate final distribution list for token dispersal"""
distribution = self.calculate_distribution(total_amount)
qualified = self.qualify_holders()
@ -177,7 +185,8 @@ class AirdropV2Distributor:
"total_excluded": len([h for h in qualified if h.is_blacklisted]),
"distribution_type": "proportional",
"recipients": [
{"address": addr, "amount": round(amount, 6), "chain": chain} for addr, amount in distribution.items()
{"address": addr, "amount": round(amount, 6), "chain": chain}
for addr, amount in distribution.items()
],
"excluded": [
{"address": h.address, "reason": h.blacklist_reason, "chain": h.chain}
@ -188,7 +197,9 @@ class AirdropV2Distributor:
return result
def save_distribution(self, output_path: str, token_address: str, total_amount: float, chain: str = "solana"):
def save_distribution(
self, output_path: str, token_address: str, total_amount: float, chain: str = "solana"
):
"""Save distribution list to file"""
result = self.generate_distribution_list(token_address, total_amount, chain)
json.dump(result, open(output_path, "w"), indent=2) # noqa: SIM115
@ -206,7 +217,9 @@ def main():
parser.add_argument("--chain", default="solana", choices=["solana", "base", "ethereum"])
parser.add_argument("--output", default="/root/backend/data/airdrop_v2_distribution.json")
parser.add_argument("--min-hold", type=float, default=0, help="Minimum holding to qualify")
parser.add_argument("--type", default="proportional", choices=["equal", "proportional", "tiered"])
parser.add_argument(
"--type", default="proportional", choices=["equal", "proportional", "tiered"]
)
args = parser.parse_args()

View file

@ -5,6 +5,7 @@ Fully automated: LLM writes → LLM reviews → publishes."""
import asyncio
import json
import logging
import os
from datetime import UTC, datetime
@ -21,13 +22,21 @@ async def fetch_market_data() -> dict:
"""Gather all market data for the report."""
async with httpx.AsyncClient(timeout=30) as c:
# Top tokens
trending = await c.get(f"{BACKEND}/api/v1/databus/fetch/trending?limit=20", headers={"X-RMI-Key": RMI_KEY})
trending = await c.get(
f"{BACKEND}/api/v1/databus/fetch/trending?limit=20", headers={"X-RMI-Key": RMI_KEY}
)
# Market overview
market = await c.get(f"{BACKEND}/api/v1/databus/fetch/market_overview", headers={"X-RMI-Key": RMI_KEY})
market = await c.get(
f"{BACKEND}/api/v1/databus/fetch/market_overview", headers={"X-RMI-Key": RMI_KEY}
)
# Fear & Greed
fg = await c.get(f"{BACKEND}/api/v1/databus/fetch/fear_greed", headers={"X-RMI-Key": RMI_KEY})
fg = await c.get(
f"{BACKEND}/api/v1/databus/fetch/fear_greed", headers={"X-RMI-Key": RMI_KEY}
)
# News
news = await c.get(f"{BACKEND}/api/v1/databus/fetch/news?limit=10", headers={"X-RMI-Key": RMI_KEY})
news = await c.get(
f"{BACKEND}/api/v1/databus/fetch/news?limit=10", headers={"X-RMI-Key": RMI_KEY}
)
return {
"trending": trending.json() if trending.status_code == 200 else {},
@ -79,7 +88,7 @@ Keep under 500 words. Professional tone. Include data where available."""
if r.status_code == 200:
return r.json().get("response", "")
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
# Fallback: basic report from data
return f"""# RMI Daily Crypto Research - {datetime.now(UTC).strftime("%B %d, %Y")}

View file

@ -22,6 +22,7 @@ import asyncio
import logging
import os
import random
import secrets
import sys
logging.basicConfig(level=logging.INFO)
@ -132,11 +133,11 @@ async def mine_hard_negatives(
if content and len(content) > 50:
negatives.append(content[:500])
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
# Fall back to static hard negatives
while len(negatives) < num_negatives:
neg = random.choice(HARD_NEGATIVES)
neg = secrets.choice(HARD_NEGATIVES)
if neg not in negatives:
negatives.append(neg)
@ -176,9 +177,9 @@ async def build_training_data() -> list[tuple[str, str, str, float]]:
# Margin: how much better is the positive?
margin = pos_score - neg_score
except Exception:
margin = random.uniform(1.5, 3.0)
margin = secrets.SystemRandom().uniform(1.5, 3.0)
else:
margin = random.uniform(1.5, 3.0)
margin = secrets.SystemRandom().uniform(1.5, 3.0)
training_data.append((query, pos_doc, neg_doc, margin))

132
scripts/fix_s112.py Normal file
View file

@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Fix S112 (try-except-continue) by logging the exception before continuing."""
from __future__ import annotations
import json
import re
import subprocess
import sys
from pathlib import Path
LOG_LINE = 'logging.getLogger(__name__).warning("swallowed exception", exc_info=True)'
SAME_LINE_CONTINUE = re.compile(
r"^(?P<indent>[ \t]*)except\b(?P<expr>[^:\n]*):\s*continue\s*(?:#.*)?$",
re.MULTILINE,
)
def ensure_import(text: str, module: str) -> str:
if re.search(rf"(?:^|\n)import\s+{re.escape(module)}(?:\s|$)", text):
return text
if re.search(rf"(?:^|\n)from\s+{re.escape(module)}\s+import", text):
return text
lines = text.splitlines(keepends=True)
insert = 0
paren = 0
for i, ln in enumerate(lines):
paren += ln.count("(") - ln.count(")")
if ln.startswith("#!/") or ln.startswith("# -*-") or ln.startswith("# coding"):
insert = i + 1
continue
if ln.startswith("from __future__ import"):
insert = i + 1
continue
if paren == 0 and (ln.startswith("import ") or ln.startswith("from ")):
insert = i + 1
if insert == len(lines) and lines and not lines[-1].endswith("\n"):
lines.append("\n")
lines.insert(insert, f"import {module}\n")
return "".join(lines)
def fix_file(path: Path) -> int:
text = path.read_text()
orig = text
# same-line: except X: continue
text = SAME_LINE_CONTINUE.sub(
lambda m: (
f"{m.group('indent')}except {m.group('expr').strip()}:\n"
f"{m.group('indent')} {LOG_LINE}\n"
f"{m.group('indent')} continue"
),
text,
)
if text != orig:
text = ensure_import(text, "logging")
path.write_text(text)
return 1
return 0
if __name__ == "__main__":
proc = subprocess.run(
[
sys.executable,
"-m",
"ruff",
"check",
"--select",
"S112",
"--output-format",
"json",
"app/",
],
capture_output=True,
text=True,
)
errors = json.loads(proc.stdout or "[]")
by_file: dict[str, list[int]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e["location"]["row"])
total_files = 0
for fn, rows in by_file.items():
path = Path(fn)
text = path.read_text()
orig = text
text = ensure_import(text, "logging")
lines = text.splitlines(keepends=True)
for row in sorted(rows, reverse=True):
if row < 1 or row > len(lines):
continue
idx = row - 1
line = lines[idx]
m = SAME_LINE_CONTINUE.match(line)
if m:
indent = m.group("indent")
expr = m.group("expr").strip()
new_lines = [
f"{indent}except {expr}:\n",
f"{indent} {LOG_LINE}\n",
f"{indent} continue\n",
]
lines[idx : idx + 1] = new_lines
continue
# multiline: find next continue
j = idx + 1
while j < len(lines):
content = lines[j].strip()
if content == "" or content.startswith("#"):
j += 1
continue
if (
content == "continue"
or content.startswith("continue ")
or content.startswith("continue\t")
):
indent = len(lines[j]) - len(lines[j].lstrip())
lines[j] = " " * indent + LOG_LINE + "\n" + " " * indent + "continue\n"
break
break
text = "".join(lines)
if text != orig:
path.write_text(text)
total_files += 1
print(f"S112: fixed {len(errors)} errors in {total_files} files")

View file

@ -1,23 +1,41 @@
#!/usr/bin/env python3
"""Bulk-fix S110, S324, and S311 ruff violations in app/."""
"""Bulk-fix S110, S324, and S311 ruff violations."""
from __future__ import annotations
import argparse
import json
import os
import py_compile
import re
import subprocess
import sys
import py_compile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
APP = ROOT / "app"
DEFAULT_TARGETS = [ROOT / "app"]
parser = argparse.ArgumentParser(description="Fix S110/S324/S311 ruff violations")
parser.add_argument("targets", nargs="*", type=Path, default=DEFAULT_TARGETS)
ARGS = parser.parse_args()
TARGETS = [t if t.is_absolute() else ROOT / t for t in ARGS.targets]
S110_LOG_LINE = 'logging.getLogger(__name__).warning("swallowed exception", exc_info=True)'
def run_ruff(select: str) -> list[dict]:
cmd = [sys.executable, "-m", "ruff", "check", "--select", select, "--output-format", "json", str(APP)]
def run_ruff(select: str, targets: list[Path] | None = None) -> list[dict]:
targets = targets or TARGETS
cmd = [
sys.executable,
"-m",
"ruff",
"check",
"--select",
select,
"--output-format",
"json",
*[str(t) for t in targets],
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode not in (0, 1):
print(f"ruff {select} failed: {result.stderr}", file=sys.stderr)
@ -72,8 +90,8 @@ SAME_LINE_PASS = re.compile(
)
def fix_s110() -> tuple[int, int]:
errors = run_ruff("S110")
def fix_s110(targets: list[Path] | None = None) -> tuple[int, int]:
errors = run_ruff("S110", targets)
by_file: dict[str, list[int]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e["location"]["row"])
@ -132,8 +150,8 @@ NEW_MD5_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])md5\1\s*\)")
NEW_SHA1_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])sha1\1\s*\)")
def fix_s324() -> tuple[int, int]:
errors = run_ruff("S324")
def fix_s324(targets: list[Path] | None = None) -> tuple[int, int]:
errors = run_ruff("S324", targets)
by_file: dict[str, list[dict]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e)
@ -145,8 +163,8 @@ def fix_s324() -> tuple[int, int]:
orig = text
text = MD5_RE.sub("hashlib.sha256(", text)
text = SHA1_RE.sub("hashlib.sha256(", text)
text = NEW_MD5_RE.sub(r'hashlib.new(\1sha256\1)', text)
text = NEW_SHA1_RE.sub(r'hashlib.new(\1sha256\1)', text)
text = NEW_MD5_RE.sub(r"hashlib.new(\1sha256\1)", text)
text = NEW_SHA1_RE.sub(r"hashlib.new(\1sha256\1)", text)
if text != orig:
path.write_text(text)
changed += 1
@ -161,8 +179,8 @@ RANDOM_CALL_RE = re.compile(r"random\.((?:random|randint|uniform|randrange))\s*\
RANDOM_CHOICE_RE = re.compile(r"random\.choice\s*\(")
def fix_s311() -> tuple[int, int]:
errors = run_ruff("S311")
def fix_s311(targets: list[Path] | None = None) -> tuple[int, int]:
errors = run_ruff("S311", targets)
by_file: dict[str, list[dict]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e)
@ -202,9 +220,10 @@ def verify_changed(files: list[Path]) -> list[str]:
def main() -> None:
os.chdir(ROOT)
s110_n, s110_files = fix_s110()
s324_n, s324_files = fix_s324()
s311_n, s311_files = fix_s311()
targets = TARGETS
s110_n, s110_files = fix_s110(targets)
s324_n, s324_files = fix_s324(targets)
s311_n, s311_files = fix_s311(targets)
print(f"S110: {s110_n} errors -> touched {s110_files} files")
print(f"S324: {s324_n} errors -> touched {s324_files} files")
print(f"S311: {s311_n} errors -> touched {s311_files} files")
@ -212,7 +231,9 @@ def main() -> None:
# compile check every touched file
changed = set()
# collect all files that were touched via git status
proc = subprocess.run(["git", "status", "--short", str(APP)], capture_output=True, text=True)
proc = subprocess.run(
["git", "status", "--short"] + [str(t) for t in targets], capture_output=True, text=True
)
for ln in proc.stdout.splitlines():
if ln.startswith(" M ") or ln.startswith("M "):
changed.add(ROOT / ln[3:].strip())

View file

@ -0,0 +1,230 @@
#!/usr/bin/env python3
"""Bulk-fix S110, S324, and S311 ruff violations in app/."""
from __future__ import annotations
import json
import os
import py_compile
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
APP = ROOT / "app"
S110_LOG_LINE = 'logging.getLogger(__name__).warning("swallowed exception", exc_info=True)'
def run_ruff(select: str) -> list[dict]:
cmd = [sys.executable, "-m", "ruff", "check", "--select", select, "--output-format", "json", str(APP)]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode not in (0, 1):
print(f"ruff {select} failed: {result.stderr}", file=sys.stderr)
return []
return json.loads(result.stdout or "[]")
def ensure_import(text: str, module: str) -> str:
"""Add `import module` at the top-level import block if missing."""
if re.search(rf"(?:^|\n)import\s+{re.escape(module)}(?:\s|$)", text):
return text
if re.search(rf"(?:^|\n)from\s+{re.escape(module)}\s+import", text):
return text
lines = text.splitlines(keepends=True)
insert = 0
paren = 0
for i, ln in enumerate(lines):
# track parentheses so we don't insert inside a multi-line parenthesized import
paren += ln.count("(") - ln.count(")")
if ln.startswith("#!/") or ln.startswith("# -*-") or ln.startswith("# coding"):
insert = i + 1
continue
if ln.startswith("from __future__ import"):
insert = i + 1
continue
if paren == 0 and (ln.startswith("import ") or ln.startswith("from ")):
insert = i + 1
if insert == len(lines) and lines and not lines[-1].endswith("\n"):
lines.append("\n")
lines.insert(insert, f"import {module}\n")
return "".join(lines)
def remove_import(text: str, module: str) -> str:
"""Remove a top-level `import module` line if it exists."""
lines = text.splitlines(keepends=True)
out = []
for ln in lines:
if re.match(rf"^import\s+{re.escape(module)}\s*$", ln.strip("\n")):
continue
out.append(ln)
return "".join(out)
# ---------------------------------------------------------------------------
# S110
# ---------------------------------------------------------------------------
SAME_LINE_PASS = re.compile(
r"^(?P<indent>[ \t]*)except\b(?P<expr>[^:\n]*):(?P<space>\s*)pass\s*(?:#.*)?$",
re.MULTILINE,
)
def fix_s110() -> tuple[int, int]:
errors = run_ruff("S110")
by_file: dict[str, list[int]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e["location"]["row"])
changed = 0
for path_str, rows in by_file.items():
path = Path(path_str)
text = path.read_text()
orig = text
lines = text.splitlines(keepends=True)
# sort rows descending so line numbers stay stable
for row in sorted(rows, reverse=True):
if row < 1 or row > len(lines):
continue
idx = row - 1
line = lines[idx]
m = SAME_LINE_PASS.match(line)
if m:
indent = m.group("indent")
expr = m.group("expr").strip()
new_lines = [f"{indent}except {expr}:\n", f"{indent} {S110_LOG_LINE}\n"]
lines[idx : idx + 1] = new_lines
continue
# multiline: find the next non-comment/blank line that is a bare pass
j = idx + 1
while j < len(lines):
content = lines[j].strip()
if content == "" or content.startswith("#"):
j += 1
continue
if content == "pass" or content.startswith("pass ") or content.startswith("pass\t"):
indent = len(lines[j]) - len(lines[j].lstrip())
lines[j] = " " * indent + S110_LOG_LINE + "\n"
break
# next executable line is not a pass; give up
break
text = "".join(lines)
# ensure logging is imported after all replacements so row indices remain valid
text = ensure_import(text, "logging")
if text != orig:
path.write_text(text)
changed += 1
return len(errors), changed
# ---------------------------------------------------------------------------
# S324
# ---------------------------------------------------------------------------
MD5_RE = re.compile(r"hashlib\.md5\s*\(")
SHA1_RE = re.compile(r"hashlib\.sha1\s*\(")
NEW_MD5_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])md5\1\s*\)")
NEW_SHA1_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])sha1\1\s*\)")
def fix_s324() -> tuple[int, int]:
errors = run_ruff("S324")
by_file: dict[str, list[dict]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e)
changed = 0
for path_str in by_file:
path = Path(path_str)
text = path.read_text()
orig = text
text = MD5_RE.sub("hashlib.sha256(", text)
text = SHA1_RE.sub("hashlib.sha256(", text)
text = NEW_MD5_RE.sub(r'hashlib.new(\1sha256\1)', text)
text = NEW_SHA1_RE.sub(r'hashlib.new(\1sha256\1)', text)
if text != orig:
path.write_text(text)
changed += 1
return len(errors), changed
# ---------------------------------------------------------------------------
# S311
# ---------------------------------------------------------------------------
RANDOM_CALL_RE = re.compile(r"random\.((?:random|randint|uniform|randrange))\s*\(")
RANDOM_CHOICE_RE = re.compile(r"random\.choice\s*\(")
def fix_s311() -> tuple[int, int]:
errors = run_ruff("S311")
by_file: dict[str, list[dict]] = {}
for e in errors:
by_file.setdefault(e["filename"], []).append(e)
changed = 0
for path_str in by_file:
path = Path(path_str)
text = path.read_text()
orig = text
text = RANDOM_CALL_RE.sub(r"secrets.SystemRandom().\1(", text)
text = RANDOM_CHOICE_RE.sub("secrets.choice(", text)
if text != orig:
text = ensure_import(text, "secrets")
# remove `import random` only if no other random.* calls remain
if not re.search(r"\brandom\.\w+\s*\(", text):
text = remove_import(text, "random")
path.write_text(text)
changed += 1
return len(errors), changed
# ---------------------------------------------------------------------------
# verification
# ---------------------------------------------------------------------------
def verify_changed(files: list[Path]) -> list[str]:
bad = []
for p in files:
try:
py_compile.compile(str(p), doraise=True)
except py_compile.PyCompileError as e:
bad.append(f"{p}: {e}")
return bad
def main() -> None:
os.chdir(ROOT)
s110_n, s110_files = fix_s110()
s324_n, s324_files = fix_s324()
s311_n, s311_files = fix_s311()
print(f"S110: {s110_n} errors -> touched {s110_files} files")
print(f"S324: {s324_n} errors -> touched {s324_files} files")
print(f"S311: {s311_n} errors -> touched {s311_files} files")
# compile check every touched file
changed = set()
# collect all files that were touched via git status
proc = subprocess.run(["git", "status", "--short", str(APP)], capture_output=True, text=True)
for ln in proc.stdout.splitlines():
if ln.startswith(" M ") or ln.startswith("M "):
changed.add(ROOT / ln[3:].strip())
if changed:
bad = verify_changed(list(changed))
if bad:
print("\nPY_COMPILE ERRORS:", file=sys.stderr)
for b in bad:
print(b, file=sys.stderr)
sys.exit(1)
print(f"py_compile OK for {len(changed)} changed files")
if __name__ == "__main__":
main()

View file

@ -7,6 +7,7 @@ The generated file is checked into git so dev installs work offline.
Re-run this script only when the chain topology changes (new data type,
new provider, etc.).
"""
from pathlib import Path
SOURCE = Path("app/databus/provider_chains.py")
@ -26,8 +27,8 @@ def main():
)
body = content.split('"""', 2)[2] if '"""' in content else content
TARGET.write_text(header + body)
print("wrote {} ({} bytes)".format(TARGET, TARGET.stat().st_size))
print(f"wrote {TARGET} ({TARGET.stat().st_size} bytes)")
if __name__ == "__main__":
main()
main()

View file

@ -59,7 +59,7 @@ def _load_cooldown_state():
with open(state_file) as f:
return json.load(f)
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
return {}
@ -70,7 +70,7 @@ def _save_cooldown_state(state: dict):
with open(state_file, "w") as f:
json.dump(state, f)
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
def _should_alert(service: str) -> bool:
@ -209,7 +209,9 @@ def send_alert(results: list):
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=10) as resp:
logger.info(f"Alert sent to webhook: HTTP {resp.status} - {len(alertable)} services down")
logger.info(
f"Alert sent to webhook: HTTP {resp.status} - {len(alertable)} services down"
)
except Exception as e:
logger.error(f"Failed to send alert to webhook: {e}")
@ -269,7 +271,7 @@ def _load_prev_state():
with open(state_file) as f:
return json.load(f)
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
return {}
@ -280,7 +282,7 @@ def _save_prev_state(state: dict):
with open(state_file, "w") as f:
json.dump(state, f)
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
# ── Main ────────────────────────────────────────────────────────────
@ -313,7 +315,9 @@ def main():
REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
BASE_WORKER_URL = os.getenv("BASE_WORKER_URL", "http://localhost:8000/api/v1/x402/fallback/status")
BASE_WORKER_URL = os.getenv(
"BASE_WORKER_URL", "http://localhost:8000/api/v1/x402/fallback/status"
)
SOLANA_WORKER_URL = os.getenv("SOLANA_WORKER_URL", "")
logger.info("Running health checks...")

View file

@ -28,7 +28,9 @@ async def ingest_via_api(content: str, metadata: dict) -> bool:
import httpx
async with httpx.AsyncClient(timeout=15) as client:
resp = await client.post(RAG_API, json={"collection": COLLECTION, "content": content, "metadata": metadata})
resp = await client.post(
RAG_API, json={"collection": COLLECTION, "content": content, "metadata": metadata}
)
return resp.status_code == 200
except Exception as e:
logger.debug(f"API ingest failed: {e}")
@ -40,15 +42,21 @@ async def main():
# Try multiple endpoints and pagination
all_reports = []
for page in range(0, 5): # Up to 5 pages of 100
for page in range(5): # Up to 5 pages of 100
url = f"{CHAINABUSE_API}?limit=100&offset={page * 100}"
try:
logger.info(f"Fetching page {page + 1} from ChainAbuse...")
async with httpx.AsyncClient(timeout=30, headers={"Accept": "application/json"}) as client:
async with httpx.AsyncClient(
timeout=30, headers={"Accept": "application/json"}
) as client:
resp = await client.get(url)
if resp.status_code == 200:
data = resp.json()
reports = data if isinstance(data, list) else data.get("data", data.get("reports", []))
reports = (
data
if isinstance(data, list)
else data.get("data", data.get("reports", []))
)
if not reports:
logger.info(f"No more reports at page {page + 1}")
break
@ -77,13 +85,17 @@ async def main():
resp = await client.get(url)
if resp.status_code == 200:
data = resp.json()
reports = data if isinstance(data, list) else data.get("data", data.get("reports", []))
reports = (
data
if isinstance(data, list)
else data.get("data", data.get("reports", []))
)
if reports:
all_reports.extend(reports)
logger.info(f"Got {len(reports)} reports from alt endpoint")
break
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
if not all_reports:
logger.warning("No ChainAbuse reports accessible via API. Ingestion skipped.")

View file

@ -140,8 +140,12 @@ async def ingest_etherscan_combined() -> dict[str, int]:
with open(csv_path, newline="", encoding="utf-8") as f:
rows = list(csv.DictReader(f))
scam_rows = [r for r in rows if any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)]
wallet_rows = [r for r in rows if not any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)]
scam_rows = [
r for r in rows if any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)
]
wallet_rows = [
r for r in rows if not any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)
]
logger.info(
f"Etherscan combined: {len(rows)} total → {len(scam_rows)} scam → known_scams, {len(wallet_rows)} → wallet_profiles"
@ -177,7 +181,9 @@ async def ingest_etherscan_combined() -> dict[str, int]:
else:
collection = "wallet_profiles"
doc_id = make_doc_id("wallet_profiles", f"eth_combined:{address}:{chain}")
content = f"Etherscan labeled address: {address} ({name}) on {chain}. Category: {label}."
content = (
f"Etherscan labeled address: {address} ({name}) on {chain}. Category: {label}."
)
metadata = {
"address": address.lower(),
"label": label,
@ -297,7 +303,7 @@ async def ingest_rekt_hacks() -> dict[str, int]:
async def ingest_solana_flagged_tokens() -> dict[str, int]:
"""Download Solana token list and ingest explicitly flagged scam tokens."""
token_list_url = "https://raw.githubusercontent.com/solana-labs/token-list/main/src/tokens/solana.tokenlist.json"
token_list_url = "https://raw.githubusercontent.com/solana-labs/token-list/main/src/tokens/solana.tokenlist.json" # noqa: S105
try:
async with httpx.AsyncClient(timeout=60) as client:
@ -317,7 +323,9 @@ async def ingest_solana_flagged_tokens() -> dict[str, int]:
for t in tokens:
name_lower = (t.get("name", "") + " " + t.get("symbol", "")).lower()
tags_lower = [str(tag).lower() for tag in t.get("tags", [])]
if any(kw in name_lower for kw in scam_keywords) or any(kw in " ".join(tags_lower) for kw in scam_keywords):
if any(kw in name_lower for kw in scam_keywords) or any(
kw in " ".join(tags_lower) for kw in scam_keywords
):
flagged.append(t)
logger.info(f"Solana token list: {len(tokens)} total, {len(flagged)} flagged as scam/spam")
@ -359,7 +367,9 @@ async def ingest_solana_flagged_tokens() -> dict[str, int]:
async def ingest_metamask_domains() -> dict[str, int]:
"""Ingest MetaMask phishing domain blacklist in batches of 50."""
config_url = "https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/config.json"
config_url = (
"https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/config.json"
)
try:
async with httpx.AsyncClient(timeout=60) as client:
@ -405,7 +415,9 @@ async def ingest_metamask_domains() -> dict[str, int]:
errors += 1
if (batch_idx + 1) % 100 == 0:
logger.info(f" MetaMask: {batch_idx + 1}/{total_batches} batches | ingested: {ingested}")
logger.info(
f" MetaMask: {batch_idx + 1}/{total_batches} batches | ingested: {ingested}"
)
await asyncio.sleep(0.05)
return {"ingested": ingested, "errors": errors}
@ -602,7 +614,9 @@ async def ingest_phishdestroy() -> dict[str, int]:
errors += 1
if (batch_idx + 1) % 100 == 0:
logger.info(f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}")
logger.info(
f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}"
)
await asyncio.sleep(0.05)
except Exception as e:
logger.warning(f"PhishDestroy active_domains.json failed: {e}")
@ -646,7 +660,9 @@ async def ingest_phishdestroy() -> dict[str, int]:
errors += 1
if (batch_idx + 1) % 100 == 0:
logger.info(f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}")
logger.info(
f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}"
)
await asyncio.sleep(0.05)
except Exception as e:
logger.warning(f"PhishDestroy domains.txt failed: {e}")

View file

@ -5,6 +5,7 @@
import asyncio
import csv
import logging
import sys
import httpx
@ -46,11 +47,13 @@ async def ingest():
}
try:
r = await client.post(RAG_URL, json=payload, headers={"X-RMI-Key": "rmi-internal-2026"})
r = await client.post(
RAG_URL, json=payload, headers={"X-RMI-Key": "rmi-internal-2026"}
)
if r.status_code == 200:
count += 1
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
if (i + 1) % 500 == 0:
print(f" Progress: {i + 1}/{len(rows)} ({count} ingested)")

View file

@ -1,9 +1,13 @@
#!/usr/bin/env python3
"""Import WHALEGOD wallet labels into Wallet Memory Bank."""
import asyncio
import logging
import os
import sys
import httpx
sys.path.insert(0, "/root/backend")
os.chdir("/root/backend")
@ -18,11 +22,6 @@ sol_labels = namespace.get("SOL_LABELS", {})
print(f"WHALEGOD labels: ETH={len(eth_labels)}, SOL={len(sol_labels)}")
# Ingest into wallet_memory
import asyncio # noqa: E402
import httpx # noqa: E402
async def ingest():
count = 0
@ -52,7 +51,7 @@ async def ingest():
if r.status_code == 200:
count += 1
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
print(f"Ingested {count}/{len(eth_labels) + len(sol_labels)} WHALEGOD labels")

View file

@ -42,7 +42,9 @@ def _get_url():
def _get_key():
return os.environ.get("SUPABASE_SERVICE_KEY", "") or os.environ.get("SUPABASE_SERVICE_ROLE_KEY", "")
return os.environ.get("SUPABASE_SERVICE_KEY", "") or os.environ.get(
"SUPABASE_SERVICE_ROLE_KEY", ""
)
def _get_headers():
@ -98,18 +100,24 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
for attempt in range(MAX_RETRIES):
try:
resp = await client.post(url, json=rows, params=params, headers=_get_headers(), timeout=60)
resp = await client.post(
url, json=rows, params=params, headers=_get_headers(), timeout=60
)
if resp.status_code in (200, 201):
return len(rows)
elif resp.status_code == 500 and "timeout" in (resp.text or "").lower():
# Statement timeout - split into smaller batches
logger.warning(f"Batch {batch_num}: statement timeout (attempt {attempt + 1}), splitting")
logger.warning(
f"Batch {batch_num}: statement timeout (attempt {attempt + 1}), splitting"
)
if len(rows) > 10:
# Split into two halves
mid = len(rows) // 2
count = 0
for half in [rows[:mid], rows[mid:]]:
r2 = await client.post(url, json=half, params=params, headers=_get_headers(), timeout=90)
r2 = await client.post(
url, json=half, params=params, headers=_get_headers(), timeout=90
)
if r2.status_code in (200, 201):
count += len(half)
else:
@ -127,10 +135,14 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
count += 1
await asyncio.sleep(0.02)
except Exception:
pass
logging.getLogger(__name__).warning(
"swallowed exception", exc_info=True
)
return count
else:
logger.error(f"Batch {batch_num} insert failed: {resp.status_code} {resp.text[:300]}")
logger.error(
f"Batch {batch_num} insert failed: {resp.status_code} {resp.text[:300]}"
)
return 0
except httpx.ReadTimeout:
logger.warning(f"Batch {batch_num}: timeout (attempt {attempt + 1})")
@ -142,7 +154,9 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
return 0
async def migrate_collection(r: aioredis.Redis, client: httpx.AsyncClient, collection: str, existing_ids: set) -> int:
async def migrate_collection(
r: aioredis.Redis, client: httpx.AsyncClient, collection: str, existing_ids: set
) -> int:
"""Migrate all docs from one Redis collection to Supabase."""
idx_key = f"rag:idx:{collection}"
doc_ids = await r.smembers(idx_key)
@ -198,7 +212,9 @@ async def migrate_collection(r: aioredis.Redis, client: httpx.AsyncClient, colle
migrated += count
batch_count += 1
if batch_count % 10 == 0:
logger.info(f" Progress: {migrated}/{len(to_migrate)} migrated ({batch_count} batches)")
logger.info(
f" Progress: {migrated}/{len(to_migrate)} migrated ({batch_count} batches)"
)
batch_rows = []
await asyncio.sleep(0.1)

View file

@ -7,6 +7,7 @@ All powered by MiniMax-Text-01 ($20/mo flat, 1M context)
"""
import json
import logging
import os
import urllib.request
from datetime import UTC, datetime
@ -19,20 +20,31 @@ if not KEY:
if line.startswith("MINIMAX_API_KEY="):
KEY = line.strip().split("=", 1)[1]
break
except: pass # noqa: E701, E722
except Exception:
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
URL = "https://api.minimax.io/v1/chat/completions"
MODEL = "MiniMax-Text-01"
BACKEND = os.getenv("BACKEND_URL", "http://localhost:8000")
def call_minimax(system: str, prompt: str, max_tokens: int = 500, temp: float = 0.7) -> str:
try:
body = json.dumps({
"model": MODEL,
"messages": [{"role":"system","content":system},{"role":"user","content":prompt}],
"max_tokens": max_tokens, "temperature": temp
}).encode()
req = urllib.request.Request(URL, data=body, headers={
"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"})
body = json.dumps(
{
"model": MODEL,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": prompt},
],
"max_tokens": max_tokens,
"temperature": temp,
}
).encode()
req = urllib.request.Request(
URL,
data=body,
headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"},
)
resp = urllib.request.urlopen(req, timeout=20)
return json.loads(resp.read())["choices"][0]["message"]["content"].strip()
except Exception as e:
@ -53,6 +65,7 @@ Write an engaging crypto security thread. Format:
Keep each tweet under 280 chars. Make it punchy and shareable."""
def generate_social_thread(scam_data: str) -> str:
return call_minimax(SOCIAL_SYSTEM, f"Recent scam data:\n{scam_data[:3000]}", 400)
@ -65,9 +78,11 @@ Answer crypto security questions concisely. Be helpful, direct, accurate.
Common topics: rug pulls, honeypots, token scanning, wallet safety, scam detection.
Always mention our free scanner if relevant. Under 200 words."""
def answer_question(question: str) -> str:
return call_minimax(SUPPORT_SYSTEM, question, 250, 0.4)
# Pre-warm cache with common questions
COMMON_QUESTIONS = [
"What is a rug pull?",
@ -82,6 +97,7 @@ COMMON_QUESTIONS = [
"What are red flags in new tokens?",
]
def warm_support_cache():
"""Pre-generate answers for common questions."""
for q in COMMON_QUESTIONS:
@ -104,6 +120,7 @@ Format:
Keep it under 150 words. Use 🔴 emojis."""
def health_narrative(metrics: dict) -> str:
prompt = json.dumps(metrics)
return call_minimax(HEALTH_SYSTEM, prompt, 250, 0.3)
@ -114,6 +131,7 @@ def health_narrative(metrics: dict) -> str:
# ═══════════════════════════════════════════
if __name__ == "__main__":
import sys
cmd = sys.argv[1] if len(sys.argv) > 1 else "all"
if cmd in ("social", "all"):
@ -134,8 +152,12 @@ if __name__ == "__main__":
print("\n=== HEALTH NARRATIVE ===")
metrics = {
"timestamp": datetime.now(UTC).isoformat(),
"backend": "alive", "disk_pct": 83, "load": "4.2",
"crons_ok": 35, "crons_total": 56, "rag_docs": 20985,
"backend": "alive",
"disk_pct": 83,
"load": "4.2",
"crons_ok": 35,
"crons_total": 56,
"rag_docs": 20985,
}
report = health_narrative(metrics)
print(report[:400])

View file

@ -88,9 +88,19 @@ async def run():
if os.path.exists(csv_path):
with open(csv_path, newline="", encoding="utf-8") as f:
rows = list(csv.DictReader(f))
scam_rows = [r for r in rows if any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)]
wallet_rows = [r for r in rows if not any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)]
logger.info(f"2. Etherscan combined: {len(rows)} total → {len(scam_rows)} scam, {len(wallet_rows)} wallets")
scam_rows = [
r
for r in rows
if any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)
]
wallet_rows = [
r
for r in rows
if not any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)
]
logger.info(
f"2. Etherscan combined: {len(rows)} total → {len(scam_rows)} scam, {len(wallet_rows)} wallets"
)
ok_scam = ok_wallet = err = 0
# Scam addresses first
@ -146,9 +156,11 @@ async def run():
if resp.status_code == 200:
ok_wallet += len(batch)
except: # noqa: E722
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
if (i // batch_size + 1) % 20 == 0:
logger.info(f" Wallets: {i + len(batch)}/{min(len(wallet_rows), 5000)} | ok={ok_wallet}")
logger.info(
f" Wallets: {i + len(batch)}/{min(len(wallet_rows), 5000)} | ok={ok_wallet}"
)
await asyncio.sleep(0.1)
total_ingested += ok_scam + ok_wallet
@ -186,7 +198,7 @@ async def run():
if resp.status_code == 200:
ok += 1
except: # noqa: E722
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
await asyncio.sleep(0.15)
total_ingested += ok
logger.info(f" Done: {ok} ingested")
@ -218,7 +230,7 @@ async def run():
if resp.status_code == 200:
ok += 1
except: # noqa: E722
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
await asyncio.sleep(0.1)
total_ingested += ok
logger.info(f" Done: {ok} ingested")

View file

@ -144,7 +144,7 @@ def _check_redis_pressure() -> bool:
return False
return True
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
return True # If check fails, proceed anyway
@ -186,12 +186,16 @@ def parse_cryptoscamdb(data: dict) -> list[dict]:
entries = data if isinstance(data, list) else data.get("entries", data.get("scams", []))
docs = []
for entry in entries[:MAX_DOCS_PER_CYCLE]:
addr = entry if isinstance(entry, str) else entry.get("address", entry.get("id", str(entry)))
addr = (
entry if isinstance(entry, str) else entry.get("address", entry.get("id", str(entry)))
)
name = entry.get("name", "") if isinstance(entry, dict) else ""
docs.append(
{
"collection": "known_scams",
"content": f"CryptoScamDB: {name} - {addr}" if name else f"CryptoScamDB entry: {addr}",
"content": f"CryptoScamDB: {name} - {addr}"
if name
else f"CryptoScamDB entry: {addr}",
"metadata": {"address": str(addr), "source": "cryptoscamdb", "name": name},
}
)
@ -302,7 +306,7 @@ async def run_unified_ingest():
timeout=5,
)
except Exception:
pass
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
if not _check_redis_pressure():
logger.warning("Redis memory pressure - skipping cycle")